1 /* This program generates documents of random lengths with random words */
2 /* using the frequencies of words and document lengths obtained from */
3 /* the Wall Street Journal(1985 - 1990) in order to create databases */
4 /* with similar stastitical properties to real text. Databases can be */
5 /* created in mg or atlas format. */
7 /* Author: D. Psaradellis, December 1994 */
9 /* Modified by J. Zobel, July 1995. */
13 * Copyright RMIT and The University of Melbourne. This code may not be
14 * further distributed without the permission of J. Zobel.
25 #define INITIAL_SEED 73802 /* initialise seed */
26 #define linelen 78 /* line length */
30 char *word; /* word */
31 int freq; /* word's frequency */
32 int cum_freq; /* cummulative freq - sum of previous
33 * freq and cum_freq */
34 short word_len; /* length of word in bytes */
38 int doc_len;/* document length */
39 int doc_freq; /* document frequency */
40 int doc_cfreq; /* cummulative frequency */
43 static struct word_info *a;
44 static struct doc_info *d;
45 static int no_of_words = 0;
46 static int word_occur = 0;
47 static int no_of_docs = 0;
48 static int doc_occur = 0;
49 static char buf [BUFLEN];
50 static int isInited = 0;
52 /* This function calculates the no. of words/doclens in the file, */
53 /* by counting the no. of newlines */
55 no_newline(char *filename)
60 if ((fp = fopen(filename, "r")) == NULL) {
61 fprintf(stderr,"Cannot open %s\n", filename);
64 while ((c = getc(fp)) != EOF)
71 /* This function dynamically allocates memory to store the words, their */
72 /* frequency, their cummulative frequency and their length in bytes */
74 build_word_array(char *filename)
77 int i , temp_cfreq = 0;
79 if ((fp = fopen(filename, "r")) == NULL) {
80 fprintf(stderr,"Cannot open %s\n", filename);
83 a = (struct word_info *)malloc(no_of_words * sizeof(struct word_info));
85 fprintf(stderr,"Can't allocate %d bytes\n", no_of_words * sizeof(struct word_info));
88 for (i = 0; i < no_of_words; i++) {
89 fscanf(fp, "%s", buf); /* store word in temporary buffer */
90 a[i].word = strdup(buf);
92 fprintf(stderr,"strdup failed\n");
95 fscanf(fp, "%d", &a[i].freq);
96 a[i].word_len = strlen(a[i].word);
97 a[i].cum_freq = temp_cfreq;
98 temp_cfreq += a[i].freq;
104 /* This function dynamically allocates memory to store the document lengths, */
105 /* their frequency and their cummulative frequency */
107 build_doc_array(char *filename)
110 int i , temp_cfreq = 0;
112 if ((fp = fopen(filename, "r")) == NULL) {
113 fprintf(stderr,"Cannot open %s\n", filename);
116 d = (struct doc_info *)malloc(no_of_docs * sizeof(struct doc_info));
117 for (i = 0; i < no_of_docs; i++) {
118 fscanf(fp, "%d", &d[i].doc_len);
119 fscanf(fp, "%d", &d[i].doc_freq);
120 d[i].doc_cfreq = temp_cfreq;
121 temp_cfreq += d[i].doc_freq;
127 /* to locate the index of the word required */
129 binsearch_word(int v)
137 if (v < a[x].cum_freq)
139 else if (v >= a[x].cum_freq + a[x].freq)
148 /* This function outputs a specified no. of random words taking into */
149 /* account the linelen */
151 output_words(StringBuf *b, int words_remain)
153 int index , linesize = 0;
155 while (words_remain > 0) { /* generate random no. in range of */
156 index = binsearch_word(rnd() % word_occur); /* 0 - word occurrences */
157 if ((a[index].word_len + linesize) > linelen) { /* if len of line
162 sb_add(b, a[index].word, a[index].word_len);
164 linesize += (a[index].word_len + 1);
173 int index = binsearch_word(rnd() % word_occur);
174 return a[index].word;
178 /* This function uses a binary search algorithm on the doc_cfreq field */
179 /* to locate the index of the document length required */
189 if (v < d[x].doc_cfreq)
191 else if (v >= d[x].doc_cfreq + d[x].doc_freq)
200 /* This function uses a binary search algorithm on the cum_freq field */
202 generate_doc(StringBuf *b) {
206 index = binsearch_doc(rnd() % doc_occur);
207 output_words( b, d[index].doc_len );
212 generate_querywords() {
217 index = binsearch_doc(rnd() % doc_occur);
218 res = (char**) malloc(sizeof(char*) * (d[index].doc_len+1));
220 for(i=0;i<d[index].doc_len;i++)
221 res[i] = get_words();
229 finnegan_init(char *lex_file, char *doc_file) {
231 fprintf(stderr,"finnegan is already inited\n");
235 printf("Initialize text generator with:\n");
236 printf("\tfile '%s' - lexeme's distribution\n", lex_file);
237 printf("\tfile '%s' - length's distribution\n", doc_file);
239 no_of_words = no_newline(lex_file);
240 no_of_docs = no_newline(doc_file);
241 word_occur = build_word_array(lex_file);
242 doc_occur = build_doc_array(doc_file);
245 /* output_docs(doc_occur, word_occur); */