| 1 |
From: Paulo Marques <pmarques@grupopie.com>
|
| 2 |
Date: Tue, 6 Sep 2005 22:16:31 +0000 (-0700)
|
| 3 |
Subject: [PATCH] kallsyms: change compression algorithm
|
| 4 |
X-Git-Url: http://www.kernel.org/git/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commitdiff;h=b3dbb4ecd46767b621df3dedd28788da93ee0cac
|
| 5 |
|
| 6 |
[PATCH] kallsyms: change compression algorithm
|
| 7 |
|
| 8 |
This patch changes the way the compression algorithm works. The base
|
| 9 |
algorithm is similiar to the previous but we force the compressed token
|
| 10 |
size to 2.
|
| 11 |
|
| 12 |
Having a fixed size compressed token allows for a lot of optimizations, and
|
| 13 |
that in turn allows this code to run over *all* the symbols faster than it
|
| 14 |
did before over just a subset.
|
| 15 |
|
| 16 |
Having it work over all the symbols will make it behave better when symbols
|
| 17 |
change positions between passes, and the "inconsistent kallsyms" messages
|
| 18 |
should become less frequent.
|
| 19 |
|
| 20 |
In my tests the compression ratio was degraded by about 0.5%, but the
|
| 21 |
results will depend greatly on the number of symbols to compress.
|
| 22 |
|
| 23 |
Signed-off-by: Paulo Marques <pmarques@grupopie.com>
|
| 24 |
Signed-off-by: Andrew Morton <akpm@osdl.org>
|
| 25 |
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
|
| 26 |
---
|
| 27 |
|
| 28 |
--- a/scripts/kallsyms.c
|
| 29 |
+++ b/scripts/kallsyms.c
|
| 30 |
@@ -24,75 +24,37 @@
|
| 31 |
*
|
| 32 |
*/
|
| 33 |
|
| 34 |
+#define _GNU_SOURCE
|
| 35 |
+
|
| 36 |
#include <stdio.h>
|
| 37 |
#include <stdlib.h>
|
| 38 |
#include <string.h>
|
| 39 |
#include <ctype.h>
|
| 40 |
|
| 41 |
-/* maximum token length used. It doesn't pay to increase it a lot, because
|
| 42 |
- * very long substrings probably don't repeat themselves too often. */
|
| 43 |
-#define MAX_TOK_SIZE 11
|
| 44 |
#define KSYM_NAME_LEN 127
|
| 45 |
|
| 46 |
-/* we use only a subset of the complete symbol table to gather the token count,
|
| 47 |
- * to speed up compression, at the expense of a little compression ratio */
|
| 48 |
-#define WORKING_SET 1024
|
| 49 |
-
|
| 50 |
-/* first find the best token only on the list of tokens that would profit more
|
| 51 |
- * than GOOD_BAD_THRESHOLD. Only if this list is empty go to the "bad" list.
|
| 52 |
- * Increasing this value will put less tokens on the "good" list, so the search
|
| 53 |
- * is faster. However, if the good list runs out of tokens, we must painfully
|
| 54 |
- * search the bad list. */
|
| 55 |
-#define GOOD_BAD_THRESHOLD 10
|
| 56 |
-
|
| 57 |
-/* token hash parameters */
|
| 58 |
-#define HASH_BITS 18
|
| 59 |
-#define HASH_TABLE_SIZE (1 << HASH_BITS)
|
| 60 |
-#define HASH_MASK (HASH_TABLE_SIZE - 1)
|
| 61 |
-#define HASH_BASE_OFFSET 2166136261U
|
| 62 |
-#define HASH_FOLD(a) ((a)&(HASH_MASK))
|
| 63 |
-
|
| 64 |
-/* flags to mark symbols */
|
| 65 |
-#define SYM_FLAG_VALID 1
|
| 66 |
-#define SYM_FLAG_SAMPLED 2
|
| 67 |
|
| 68 |
struct sym_entry {
|
| 69 |
unsigned long long addr;
|
| 70 |
- char type;
|
| 71 |
- unsigned char flags;
|
| 72 |
- unsigned char len;
|
| 73 |
+ unsigned int len;
|
| 74 |
unsigned char *sym;
|
| 75 |
};
|
| 76 |
|
| 77 |
|
| 78 |
static struct sym_entry *table;
|
| 79 |
-static int size, cnt;
|
| 80 |
+static unsigned int table_size, table_cnt;
|
| 81 |
static unsigned long long _stext, _etext, _sinittext, _einittext, _sextratext, _eextratext;
|
| 82 |
static int all_symbols = 0;
|
| 83 |
static char symbol_prefix_char = '\0';
|
| 84 |
|
| 85 |
-struct token {
|
| 86 |
- unsigned char data[MAX_TOK_SIZE];
|
| 87 |
- unsigned char len;
|
| 88 |
- /* profit: the number of bytes that could be saved by inserting this
|
| 89 |
- * token into the table */
|
| 90 |
- int profit;
|
| 91 |
- struct token *next; /* next token on the hash list */
|
| 92 |
- struct token *right; /* next token on the good/bad list */
|
| 93 |
- struct token *left; /* previous token on the good/bad list */
|
| 94 |
- struct token *smaller; /* token that is less one letter than this one */
|
| 95 |
- };
|
| 96 |
-
|
| 97 |
-struct token bad_head, good_head;
|
| 98 |
-struct token *hash_table[HASH_TABLE_SIZE];
|
| 99 |
+int token_profit[0x10000];
|
| 100 |
|
| 101 |
/* the table that holds the result of the compression */
|
| 102 |
-unsigned char best_table[256][MAX_TOK_SIZE+1];
|
| 103 |
+unsigned char best_table[256][2];
|
| 104 |
unsigned char best_table_len[256];
|
| 105 |
|
| 106 |
|
| 107 |
-static void
|
| 108 |
-usage(void)
|
| 109 |
+static void usage(void)
|
| 110 |
{
|
| 111 |
fprintf(stderr, "Usage: kallsyms [--all-symbols] [--symbol-prefix=<prefix char>] < in.map > out.S\n");
|
| 112 |
exit(1);
|
| 113 |
@@ -102,21 +64,19 @@ usage(void)
|
| 114 |
* This ignores the intensely annoying "mapping symbols" found
|
| 115 |
* in ARM ELF files: $a, $t and $d.
|
| 116 |
*/
|
| 117 |
-static inline int
|
| 118 |
-is_arm_mapping_symbol(const char *str)
|
| 119 |
+static inline int is_arm_mapping_symbol(const char *str)
|
| 120 |
{
|
| 121 |
return str[0] == '$' && strchr("atd", str[1])
|
| 122 |
&& (str[2] == '\0' || str[2] == '.');
|
| 123 |
}
|
| 124 |
|
| 125 |
-static int
|
| 126 |
-read_symbol(FILE *in, struct sym_entry *s)
|
| 127 |
+static int read_symbol(FILE *in, struct sym_entry *s)
|
| 128 |
{
|
| 129 |
char str[500];
|
| 130 |
- char *sym;
|
| 131 |
+ char *sym, stype;
|
| 132 |
int rc;
|
| 133 |
|
| 134 |
- rc = fscanf(in, "%llx %c %499s\n", &s->addr, &s->type, str);
|
| 135 |
+ rc = fscanf(in, "%llx %c %499s\n", &s->addr, &stype, str);
|
| 136 |
if (rc != 3) {
|
| 137 |
if (rc != EOF) {
|
| 138 |
/* skip line */
|
| 139 |
@@ -143,7 +103,7 @@ read_symbol(FILE *in, struct sym_entry *
|
| 140 |
_sextratext = s->addr;
|
| 141 |
else if (strcmp(sym, "_eextratext") == 0)
|
| 142 |
_eextratext = s->addr;
|
| 143 |
- else if (toupper(s->type) == 'A')
|
| 144 |
+ else if (toupper(stype) == 'A')
|
| 145 |
{
|
| 146 |
/* Keep these useful absolute symbols */
|
| 147 |
if (strcmp(sym, "__kernel_syscall_via_break") &&
|
| 148 |
@@ -153,22 +113,21 @@ read_symbol(FILE *in, struct sym_entry *
|
| 149 |
return -1;
|
| 150 |
|
| 151 |
}
|
| 152 |
- else if (toupper(s->type) == 'U' ||
|
| 153 |
+ else if (toupper(stype) == 'U' ||
|
| 154 |
is_arm_mapping_symbol(sym))
|
| 155 |
return -1;
|
| 156 |
|
| 157 |
/* include the type field in the symbol name, so that it gets
|
| 158 |
* compressed together */
|
| 159 |
s->len = strlen(str) + 1;
|
| 160 |
- s->sym = (char *) malloc(s->len + 1);
|
| 161 |
- strcpy(s->sym + 1, str);
|
| 162 |
- s->sym[0] = s->type;
|
| 163 |
+ s->sym = malloc(s->len + 1);
|
| 164 |
+ strcpy((char *)s->sym + 1, str);
|
| 165 |
+ s->sym[0] = stype;
|
| 166 |
|
| 167 |
return 0;
|
| 168 |
}
|
| 169 |
|
| 170 |
-static int
|
| 171 |
-symbol_valid(struct sym_entry *s)
|
| 172 |
+static int symbol_valid(struct sym_entry *s)
|
| 173 |
{
|
| 174 |
/* Symbols which vary between passes. Passes 1 and 2 must have
|
| 175 |
* identical symbol lists. The kallsyms_* symbols below are only added
|
| 176 |
@@ -214,30 +173,29 @@ symbol_valid(struct sym_entry *s)
|
| 177 |
}
|
| 178 |
|
| 179 |
/* Exclude symbols which vary between passes. */
|
| 180 |
- if (strstr(s->sym + offset, "_compiled."))
|
| 181 |
+ if (strstr((char *)s->sym + offset, "_compiled."))
|
| 182 |
return 0;
|
| 183 |
|
| 184 |
for (i = 0; special_symbols[i]; i++)
|
| 185 |
- if( strcmp(s->sym + offset, special_symbols[i]) == 0 )
|
| 186 |
+ if( strcmp((char *)s->sym + offset, special_symbols[i]) == 0 )
|
| 187 |
return 0;
|
| 188 |
|
| 189 |
return 1;
|
| 190 |
}
|
| 191 |
|
| 192 |
-static void
|
| 193 |
-read_map(FILE *in)
|
| 194 |
+static void read_map(FILE *in)
|
| 195 |
{
|
| 196 |
while (!feof(in)) {
|
| 197 |
- if (cnt >= size) {
|
| 198 |
- size += 10000;
|
| 199 |
- table = realloc(table, sizeof(*table) * size);
|
| 200 |
+ if (table_cnt >= table_size) {
|
| 201 |
+ table_size += 10000;
|
| 202 |
+ table = realloc(table, sizeof(*table) * table_size);
|
| 203 |
if (!table) {
|
| 204 |
fprintf(stderr, "out of memory\n");
|
| 205 |
exit (1);
|
| 206 |
}
|
| 207 |
}
|
| 208 |
- if (read_symbol(in, &table[cnt]) == 0)
|
| 209 |
- cnt++;
|
| 210 |
+ if (read_symbol(in, &table[table_cnt]) == 0)
|
| 211 |
+ table_cnt++;
|
| 212 |
}
|
| 213 |
}
|
| 214 |
|
| 215 |
@@ -281,10 +239,9 @@ static int expand_symbol(unsigned char *
|
| 216 |
return total;
|
| 217 |
}
|
| 218 |
|
| 219 |
-static void
|
| 220 |
-write_src(void)
|
| 221 |
+static void write_src(void)
|
| 222 |
{
|
| 223 |
- int i, k, off, valid;
|
| 224 |
+ unsigned int i, k, off;
|
| 225 |
unsigned int best_idx[256];
|
| 226 |
unsigned int *markers;
|
| 227 |
char buf[KSYM_NAME_LEN+1];
|
| 228 |
@@ -301,33 +258,24 @@ write_src(void)
|
| 229 |
printf(".data\n");
|
| 230 |
|
| 231 |
output_label("kallsyms_addresses");
|
| 232 |
- valid = 0;
|
| 233 |
- for (i = 0; i < cnt; i++) {
|
| 234 |
- if (table[i].flags & SYM_FLAG_VALID) {
|
| 235 |
- printf("\tPTR\t%#llx\n", table[i].addr);
|
| 236 |
- valid++;
|
| 237 |
- }
|
| 238 |
+ for (i = 0; i < table_cnt; i++) {
|
| 239 |
+ printf("\tPTR\t%#llx\n", table[i].addr);
|
| 240 |
}
|
| 241 |
printf("\n");
|
| 242 |
|
| 243 |
output_label("kallsyms_num_syms");
|
| 244 |
- printf("\tPTR\t%d\n", valid);
|
| 245 |
+ printf("\tPTR\t%d\n", table_cnt);
|
| 246 |
printf("\n");
|
| 247 |
|
| 248 |
/* table of offset markers, that give the offset in the compressed stream
|
| 249 |
* every 256 symbols */
|
| 250 |
- markers = (unsigned int *) malloc(sizeof(unsigned int)*((valid + 255) / 256));
|
| 251 |
+ markers = (unsigned int *) malloc(sizeof(unsigned int) * ((table_cnt + 255) / 256));
|
| 252 |
|
| 253 |
output_label("kallsyms_names");
|
| 254 |
- valid = 0;
|
| 255 |
off = 0;
|
| 256 |
- for (i = 0; i < cnt; i++) {
|
| 257 |
-
|
| 258 |
- if (!table[i].flags & SYM_FLAG_VALID)
|
| 259 |
- continue;
|
| 260 |
-
|
| 261 |
- if ((valid & 0xFF) == 0)
|
| 262 |
- markers[valid >> 8] = off;
|
| 263 |
+ for (i = 0; i < table_cnt; i++) {
|
| 264 |
+ if ((i & 0xFF) == 0)
|
| 265 |
+ markers[i >> 8] = off;
|
| 266 |
|
| 267 |
printf("\t.byte 0x%02x", table[i].len);
|
| 268 |
for (k = 0; k < table[i].len; k++)
|
| 269 |
@@ -335,12 +283,11 @@ write_src(void)
|
| 270 |
printf("\n");
|
| 271 |
|
| 272 |
off += table[i].len + 1;
|
| 273 |
- valid++;
|
| 274 |
}
|
| 275 |
printf("\n");
|
| 276 |
|
| 277 |
output_label("kallsyms_markers");
|
| 278 |
- for (i = 0; i < ((valid + 255) >> 8); i++)
|
| 279 |
+ for (i = 0; i < ((table_cnt + 255) >> 8); i++)
|
| 280 |
printf("\tPTR\t%d\n", markers[i]);
|
| 281 |
printf("\n");
|
| 282 |
|
| 283 |
@@ -350,7 +297,7 @@ write_src(void)
|
| 284 |
off = 0;
|
| 285 |
for (i = 0; i < 256; i++) {
|
| 286 |
best_idx[i] = off;
|
| 287 |
- expand_symbol(best_table[i],best_table_len[i],buf);
|
| 288 |
+ expand_symbol(best_table[i], best_table_len[i], buf);
|
| 289 |
printf("\t.asciz\t\"%s\"\n", buf);
|
| 290 |
off += strlen(buf) + 1;
|
| 291 |
}
|
| 292 |
@@ -365,153 +312,13 @@ write_src(void)
|
| 293 |
|
| 294 |
/* table lookup compression functions */
|
| 295 |
|
| 296 |
-static inline unsigned int rehash_token(unsigned int hash, unsigned char data)
|
| 297 |
-{
|
| 298 |
- return ((hash * 16777619) ^ data);
|
| 299 |
-}
|
| 300 |
-
|
| 301 |
-static unsigned int hash_token(unsigned char *data, int len)
|
| 302 |
-{
|
| 303 |
- unsigned int hash=HASH_BASE_OFFSET;
|
| 304 |
- int i;
|
| 305 |
-
|
| 306 |
- for (i = 0; i < len; i++)
|
| 307 |
- hash = rehash_token(hash, data[i]);
|
| 308 |
-
|
| 309 |
- return HASH_FOLD(hash);
|
| 310 |
-}
|
| 311 |
-
|
| 312 |
-/* find a token given its data and hash value */
|
| 313 |
-static struct token *find_token_hash(unsigned char *data, int len, unsigned int hash)
|
| 314 |
-{
|
| 315 |
- struct token *ptr;
|
| 316 |
-
|
| 317 |
- ptr = hash_table[hash];
|
| 318 |
-
|
| 319 |
- while (ptr) {
|
| 320 |
- if ((ptr->len == len) && (memcmp(ptr->data, data, len) == 0))
|
| 321 |
- return ptr;
|
| 322 |
- ptr=ptr->next;
|
| 323 |
- }
|
| 324 |
-
|
| 325 |
- return NULL;
|
| 326 |
-}
|
| 327 |
-
|
| 328 |
-static inline void insert_token_in_group(struct token *head, struct token *ptr)
|
| 329 |
-{
|
| 330 |
- ptr->right = head->right;
|
| 331 |
- ptr->right->left = ptr;
|
| 332 |
- head->right = ptr;
|
| 333 |
- ptr->left = head;
|
| 334 |
-}
|
| 335 |
-
|
| 336 |
-static inline void remove_token_from_group(struct token *ptr)
|
| 337 |
-{
|
| 338 |
- ptr->left->right = ptr->right;
|
| 339 |
- ptr->right->left = ptr->left;
|
| 340 |
-}
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-/* build the counts for all the tokens that start with "data", and have lenghts
|
| 344 |
- * from 2 to "len" */
|
| 345 |
-static void learn_token(unsigned char *data, int len)
|
| 346 |
-{
|
| 347 |
- struct token *ptr,*last_ptr;
|
| 348 |
- int i, newprofit;
|
| 349 |
- unsigned int hash = HASH_BASE_OFFSET;
|
| 350 |
- unsigned int hashes[MAX_TOK_SIZE + 1];
|
| 351 |
-
|
| 352 |
- if (len > MAX_TOK_SIZE)
|
| 353 |
- len = MAX_TOK_SIZE;
|
| 354 |
-
|
| 355 |
- /* calculate and store the hash values for all the sub-tokens */
|
| 356 |
- hash = rehash_token(hash, data[0]);
|
| 357 |
- for (i = 2; i <= len; i++) {
|
| 358 |
- hash = rehash_token(hash, data[i-1]);
|
| 359 |
- hashes[i] = HASH_FOLD(hash);
|
| 360 |
- }
|
| 361 |
-
|
| 362 |
- last_ptr = NULL;
|
| 363 |
- ptr = NULL;
|
| 364 |
-
|
| 365 |
- for (i = len; i >= 2; i--) {
|
| 366 |
- hash = hashes[i];
|
| 367 |
-
|
| 368 |
- if (!ptr) ptr = find_token_hash(data, i, hash);
|
| 369 |
-
|
| 370 |
- if (!ptr) {
|
| 371 |
- /* create a new token entry */
|
| 372 |
- ptr = (struct token *) malloc(sizeof(*ptr));
|
| 373 |
-
|
| 374 |
- memcpy(ptr->data, data, i);
|
| 375 |
- ptr->len = i;
|
| 376 |
-
|
| 377 |
- /* when we create an entry, it's profit is 0 because
|
| 378 |
- * we also take into account the size of the token on
|
| 379 |
- * the compressed table. We then subtract GOOD_BAD_THRESHOLD
|
| 380 |
- * so that the test to see if this token belongs to
|
| 381 |
- * the good or bad list, is a comparison to zero */
|
| 382 |
- ptr->profit = -GOOD_BAD_THRESHOLD;
|
| 383 |
-
|
| 384 |
- ptr->next = hash_table[hash];
|
| 385 |
- hash_table[hash] = ptr;
|
| 386 |
-
|
| 387 |
- insert_token_in_group(&bad_head, ptr);
|
| 388 |
-
|
| 389 |
- ptr->smaller = NULL;
|
| 390 |
- } else {
|
| 391 |
- newprofit = ptr->profit + (ptr->len - 1);
|
| 392 |
- /* check to see if this token needs to be moved to a
|
| 393 |
- * different list */
|
| 394 |
- if((ptr->profit < 0) && (newprofit >= 0)) {
|
| 395 |
- remove_token_from_group(ptr);
|
| 396 |
- insert_token_in_group(&good_head,ptr);
|
| 397 |
- }
|
| 398 |
- ptr->profit = newprofit;
|
| 399 |
- }
|
| 400 |
-
|
| 401 |
- if (last_ptr) last_ptr->smaller = ptr;
|
| 402 |
- last_ptr = ptr;
|
| 403 |
-
|
| 404 |
- ptr = ptr->smaller;
|
| 405 |
- }
|
| 406 |
-}
|
| 407 |
-
|
| 408 |
-/* decrease the counts for all the tokens that start with "data", and have lenghts
|
| 409 |
- * from 2 to "len". This function is much simpler than learn_token because we have
|
| 410 |
- * more guarantees (tho tokens exist, the ->smaller pointer is set, etc.)
|
| 411 |
- * The two separate functions exist only because of compression performance */
|
| 412 |
-static void forget_token(unsigned char *data, int len)
|
| 413 |
-{
|
| 414 |
- struct token *ptr;
|
| 415 |
- int i, newprofit;
|
| 416 |
- unsigned int hash=0;
|
| 417 |
-
|
| 418 |
- if (len > MAX_TOK_SIZE) len = MAX_TOK_SIZE;
|
| 419 |
-
|
| 420 |
- hash = hash_token(data, len);
|
| 421 |
- ptr = find_token_hash(data, len, hash);
|
| 422 |
-
|
| 423 |
- for (i = len; i >= 2; i--) {
|
| 424 |
-
|
| 425 |
- newprofit = ptr->profit - (ptr->len - 1);
|
| 426 |
- if ((ptr->profit >= 0) && (newprofit < 0)) {
|
| 427 |
- remove_token_from_group(ptr);
|
| 428 |
- insert_token_in_group(&bad_head, ptr);
|
| 429 |
- }
|
| 430 |
- ptr->profit=newprofit;
|
| 431 |
-
|
| 432 |
- ptr=ptr->smaller;
|
| 433 |
- }
|
| 434 |
-}
|
| 435 |
-
|
| 436 |
/* count all the possible tokens in a symbol */
|
| 437 |
static void learn_symbol(unsigned char *symbol, int len)
|
| 438 |
{
|
| 439 |
int i;
|
| 440 |
|
| 441 |
for (i = 0; i < len - 1; i++)
|
| 442 |
- learn_token(symbol + i, len - i);
|
| 443 |
+ token_profit[ symbol[i] + (symbol[i + 1] << 8) ]++;
|
| 444 |
}
|
| 445 |
|
| 446 |
/* decrease the count for all the possible tokens in a symbol */
|
| 447 |
@@ -520,117 +327,90 @@ static void forget_symbol(unsigned char
|
| 448 |
int i;
|
| 449 |
|
| 450 |
for (i = 0; i < len - 1; i++)
|
| 451 |
- forget_token(symbol + i, len - i);
|
| 452 |
+ token_profit[ symbol[i] + (symbol[i + 1] << 8) ]--;
|
| 453 |
}
|
| 454 |
|
| 455 |
-/* set all the symbol flags and do the initial token count */
|
| 456 |
+/* remove all the invalid symbols from the table and do the initial token count */
|
| 457 |
static void build_initial_tok_table(void)
|
| 458 |
{
|
| 459 |
- int i, use_it, valid;
|
| 460 |
+ unsigned int i, pos;
|
| 461 |
|
| 462 |
- valid = 0;
|
| 463 |
- for (i = 0; i < cnt; i++) {
|
| 464 |
- table[i].flags = 0;
|
| 465 |
+ pos = 0;
|
| 466 |
+ for (i = 0; i < table_cnt; i++) {
|
| 467 |
if ( symbol_valid(&table[i]) ) {
|
| 468 |
- table[i].flags |= SYM_FLAG_VALID;
|
| 469 |
- valid++;
|
| 470 |
+ if (pos != i)
|
| 471 |
+ table[pos] = table[i];
|
| 472 |
+ learn_symbol(table[pos].sym, table[pos].len);
|
| 473 |
+ pos++;
|
| 474 |
}
|
| 475 |
}
|
| 476 |
-
|
| 477 |
- use_it = 0;
|
| 478 |
- for (i = 0; i < cnt; i++) {
|
| 479 |
-
|
| 480 |
- /* subsample the available symbols. This method is almost like
|
| 481 |
- * a Bresenham's algorithm to get uniformly distributed samples
|
| 482 |
- * across the symbol table */
|
| 483 |
- if (table[i].flags & SYM_FLAG_VALID) {
|
| 484 |
-
|
| 485 |
- use_it += WORKING_SET;
|
| 486 |
-
|
| 487 |
- if (use_it >= valid) {
|
| 488 |
- table[i].flags |= SYM_FLAG_SAMPLED;
|
| 489 |
- use_it -= valid;
|
| 490 |
- }
|
| 491 |
- }
|
| 492 |
- if (table[i].flags & SYM_FLAG_SAMPLED)
|
| 493 |
- learn_symbol(table[i].sym, table[i].len);
|
| 494 |
- }
|
| 495 |
+ table_cnt = pos;
|
| 496 |
}
|
| 497 |
|
| 498 |
/* replace a given token in all the valid symbols. Use the sampled symbols
|
| 499 |
* to update the counts */
|
| 500 |
-static void compress_symbols(unsigned char *str, int tlen, int idx)
|
| 501 |
+static void compress_symbols(unsigned char *str, int idx)
|
| 502 |
{
|
| 503 |
- int i, len, learn, size;
|
| 504 |
- unsigned char *p;
|
| 505 |
-
|
| 506 |
- for (i = 0; i < cnt; i++) {
|
| 507 |
+ unsigned int i, len, size;
|
| 508 |
+ unsigned char *p1, *p2;
|
| 509 |
|
| 510 |
- if (!(table[i].flags & SYM_FLAG_VALID)) continue;
|
| 511 |
+ for (i = 0; i < table_cnt; i++) {
|
| 512 |
|
| 513 |
len = table[i].len;
|
| 514 |
- learn = 0;
|
| 515 |
- p = table[i].sym;
|
| 516 |
+ p1 = table[i].sym;
|
| 517 |
+
|
| 518 |
+ /* find the token on the symbol */
|
| 519 |
+ p2 = memmem(p1, len, str, 2);
|
| 520 |
+ if (!p2) continue;
|
| 521 |
+
|
| 522 |
+ /* decrease the counts for this symbol's tokens */
|
| 523 |
+ forget_symbol(table[i].sym, len);
|
| 524 |
+
|
| 525 |
+ size = len;
|
| 526 |
|
| 527 |
do {
|
| 528 |
+ *p2 = idx;
|
| 529 |
+ p2++;
|
| 530 |
+ size -= (p2 - p1);
|
| 531 |
+ memmove(p2, p2 + 1, size);
|
| 532 |
+ p1 = p2;
|
| 533 |
+ len--;
|
| 534 |
+
|
| 535 |
+ if (size < 2) break;
|
| 536 |
+
|
| 537 |
/* find the token on the symbol */
|
| 538 |
- p = (unsigned char *) strstr((char *) p, (char *) str);
|
| 539 |
- if (!p) break;
|
| 540 |
+ p2 = memmem(p1, size, str, 2);
|
| 541 |
|
| 542 |
- if (!learn) {
|
| 543 |
- /* if this symbol was used to count, decrease it */
|
| 544 |
- if (table[i].flags & SYM_FLAG_SAMPLED)
|
| 545 |
- forget_symbol(table[i].sym, len);
|
| 546 |
- learn = 1;
|
| 547 |
- }
|
| 548 |
+ } while (p2);
|
| 549 |
|
| 550 |
- *p = idx;
|
| 551 |
- size = (len - (p - table[i].sym)) - tlen + 1;
|
| 552 |
- memmove(p + 1, p + tlen, size);
|
| 553 |
- p++;
|
| 554 |
- len -= tlen - 1;
|
| 555 |
-
|
| 556 |
- } while (size >= tlen);
|
| 557 |
-
|
| 558 |
- if(learn) {
|
| 559 |
- table[i].len = len;
|
| 560 |
- /* if this symbol was used to count, learn it again */
|
| 561 |
- if(table[i].flags & SYM_FLAG_SAMPLED)
|
| 562 |
- learn_symbol(table[i].sym, len);
|
| 563 |
- }
|
| 564 |
+ table[i].len = len;
|
| 565 |
+
|
| 566 |
+ /* increase the counts for this symbol's new tokens */
|
| 567 |
+ learn_symbol(table[i].sym, len);
|
| 568 |
}
|
| 569 |
}
|
| 570 |
|
| 571 |
/* search the token with the maximum profit */
|
| 572 |
-static struct token *find_best_token(void)
|
| 573 |
+static int find_best_token(void)
|
| 574 |
{
|
| 575 |
- struct token *ptr,*best,*head;
|
| 576 |
- int bestprofit;
|
| 577 |
+ int i, best, bestprofit;
|
| 578 |
|
| 579 |
bestprofit=-10000;
|
| 580 |
+ best = 0;
|
| 581 |
|
| 582 |
- /* failsafe: if the "good" list is empty search from the "bad" list */
|
| 583 |
- if(good_head.right == &good_head) head = &bad_head;
|
| 584 |
- else head = &good_head;
|
| 585 |
-
|
| 586 |
- ptr = head->right;
|
| 587 |
- best = NULL;
|
| 588 |
- while (ptr != head) {
|
| 589 |
- if (ptr->profit > bestprofit) {
|
| 590 |
- bestprofit = ptr->profit;
|
| 591 |
- best = ptr;
|
| 592 |
+ for (i = 0; i < 0x10000; i++) {
|
| 593 |
+ if (token_profit[i] > bestprofit) {
|
| 594 |
+ best = i;
|
| 595 |
+ bestprofit = token_profit[i];
|
| 596 |
}
|
| 597 |
- ptr = ptr->right;
|
| 598 |
}
|
| 599 |
-
|
| 600 |
return best;
|
| 601 |
}
|
| 602 |
|
| 603 |
/* this is the core of the algorithm: calculate the "best" table */
|
| 604 |
static void optimize_result(void)
|
| 605 |
{
|
| 606 |
- struct token *best;
|
| 607 |
- int i;
|
| 608 |
+ int i, best;
|
| 609 |
|
| 610 |
/* using the '\0' symbol last allows compress_symbols to use standard
|
| 611 |
* fast string functions */
|
| 612 |
@@ -644,14 +424,12 @@ static void optimize_result(void)
|
| 613 |
best = find_best_token();
|
| 614 |
|
| 615 |
/* place it in the "best" table */
|
| 616 |
- best_table_len[i] = best->len;
|
| 617 |
- memcpy(best_table[i], best->data, best_table_len[i]);
|
| 618 |
- /* zero terminate the token so that we can use strstr
|
| 619 |
- in compress_symbols */
|
| 620 |
- best_table[i][best_table_len[i]]='\0';
|
| 621 |
+ best_table_len[i] = 2;
|
| 622 |
+ best_table[i][0] = best & 0xFF;
|
| 623 |
+ best_table[i][1] = (best >> 8) & 0xFF;
|
| 624 |
|
| 625 |
/* replace this token in all the valid symbols */
|
| 626 |
- compress_symbols(best_table[i], best_table_len[i], i);
|
| 627 |
+ compress_symbols(best_table[i], i);
|
| 628 |
}
|
| 629 |
}
|
| 630 |
}
|
| 631 |
@@ -659,39 +437,28 @@ static void optimize_result(void)
|
| 632 |
/* start by placing the symbols that are actually used on the table */
|
| 633 |
static void insert_real_symbols_in_table(void)
|
| 634 |
{
|
| 635 |
- int i, j, c;
|
| 636 |
+ unsigned int i, j, c;
|
| 637 |
|
| 638 |
memset(best_table, 0, sizeof(best_table));
|
| 639 |
memset(best_table_len, 0, sizeof(best_table_len));
|
| 640 |
|
| 641 |
- for (i = 0; i < cnt; i++) {
|
| 642 |
- if (table[i].flags & SYM_FLAG_VALID) {
|
| 643 |
- for (j = 0; j < table[i].len; j++) {
|
| 644 |
- c = table[i].sym[j];
|
| 645 |
- best_table[c][0]=c;
|
| 646 |
- best_table_len[c]=1;
|
| 647 |
- }
|
| 648 |
+ for (i = 0; i < table_cnt; i++) {
|
| 649 |
+ for (j = 0; j < table[i].len; j++) {
|
| 650 |
+ c = table[i].sym[j];
|
| 651 |
+ best_table[c][0]=c;
|
| 652 |
+ best_table_len[c]=1;
|
| 653 |
}
|
| 654 |
}
|
| 655 |
}
|
| 656 |
|
| 657 |
static void optimize_token_table(void)
|
| 658 |
{
|
| 659 |
- memset(hash_table, 0, sizeof(hash_table));
|
| 660 |
-
|
| 661 |
- good_head.left = &good_head;
|
| 662 |
- good_head.right = &good_head;
|
| 663 |
-
|
| 664 |
- bad_head.left = &bad_head;
|
| 665 |
- bad_head.right = &bad_head;
|
| 666 |
-
|
| 667 |
build_initial_tok_table();
|
| 668 |
|
| 669 |
insert_real_symbols_in_table();
|
| 670 |
|
| 671 |
/* When valid symbol is not registered, exit to error */
|
| 672 |
- if (good_head.left == good_head.right &&
|
| 673 |
- bad_head.left == bad_head.right) {
|
| 674 |
+ if (!table_cnt) {
|
| 675 |
fprintf(stderr, "No valid symbol.\n");
|
| 676 |
exit(1);
|
| 677 |
}
|
| 678 |
@@ -700,8 +467,7 @@ static void optimize_token_table(void)
|
| 679 |
}
|
| 680 |
|
| 681 |
|
| 682 |
-int
|
| 683 |
-main(int argc, char **argv)
|
| 684 |
+int main(int argc, char **argv)
|
| 685 |
{
|
| 686 |
if (argc >= 2) {
|
| 687 |
int i;
|
Parent Directory
| 
Revision Log