Browse Source

Add fastlz poc

optixx 8 years ago
parent
commit
a616e7ce9b
7 changed files with 1874 additions and 0 deletions
  1. 636 0
      poc/fastlz/6pack.c
  2. 478 0
      poc/fastlz/6unpack.c
  3. 5 0
      poc/fastlz/Makefile
  4. 426 0
      poc/fastlz/fastlz.c
  5. 99 0
      poc/fastlz/fastlz.h
  6. 150 0
      poc/fastlz/poc.c
  7. 80 0
      poc/fastlz/ringbuffer.h

+ 636 - 0
poc/fastlz/6pack.c

@@ -0,0 +1,636 @@
+/*
+  6PACK - file compressor using FastLZ (lightning-fast compression library)
+
+  Copyright (C) 2007 Ariya Hidayat (ariya@kde.org)
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define SIXPACK_VERSION_MAJOR    0
+#define SIXPACK_VERSION_MINOR    1
+#define SIXPACK_VERSION_REVISION 0
+#define SIXPACK_VERSION_STRING   "snapshot 20070615"
+
+#include "fastlz.h"
+
+#undef PATH_SEPARATOR
+
+#if defined(MSDOS) || defined(__MSDOS__) || defined(MSDOS)
+#define PATH_SEPARATOR '\\'
+#endif
+
+#if defined(WIN32) || defined(__NT__) || defined(_WIN32) || defined(__WIN32__)
+#define PATH_SEPARATOR '\\'
+#if defined(__BORLANDC__) || defined(_MSC_VER)
+#define inline __inline
+#endif
+#endif
+
+#ifndef PATH_SEPARATOR
+#define PATH_SEPARATOR '/'
+#endif
+
+#undef SIXPACK_BENCHMARK_WIN32
+#if defined(WIN32) || defined(__NT__) || defined(_WIN32) || defined(__WIN32__)
+#if defined(_MSC_VER) || defined(__GNUC__)
+#define SIXPACK_BENCHMARK_WIN32
+#include <windows.h>
+#endif
+#endif
+
+/* magic identifier for 6pack file */
+static unsigned char sixpack_magic[8] = {137, '6', 'P', 'K', 13, 10, 26, 10};
+
+#define BLOCK_SIZE (2*64*1024)
+
+/* prototypes */
+static inline unsigned long update_adler32(unsigned long checksum, const void *buf, int len);
+void usage(void);
+int detect_magic(FILE *f);
+void write_magic(FILE *f);
+void write_chunk_header(FILE* f, int id, int options, unsigned long size,
+unsigned long checksum, unsigned long extra);
+unsigned long block_compress(const unsigned char* input, unsigned long length, unsigned char* output);
+int pack_file_compressed(const char* input_file, int method, int level, FILE* f);
+int pack_file(int compress_level, const char* input_file, const char* output_file);
+
+/* for Adler-32 checksum algorithm, see RFC 1950 Section 8.2 */
+#define ADLER32_BASE 65521
+static inline unsigned long update_adler32(unsigned long checksum, const void *buf, int len)
+{
+  const unsigned char* ptr = (const unsigned char*)buf;
+  unsigned long s1 = checksum & 0xffff;
+  unsigned long s2 = (checksum >> 16) & 0xffff;
+
+  while(len>0)
+  {
+    unsigned k = len < 5552 ? len : 5552;
+    len -= k;
+
+    while(k >= 8)
+    {
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      k -= 8;
+    }
+
+    while(k-- > 0)
+    {
+      s1 += *ptr++; s2 += s1;
+    }
+    s1 = s1 % ADLER32_BASE;
+    s2 = s2 % ADLER32_BASE;
+  }
+  return (s2 << 16) + s1;
+}
+
+void usage(void)
+{
+  printf("6pack: high-speed file compression tool\n");
+  printf("Copyright (C) 2007 Ariya Hidayat (ariya@kde.org)\n");
+  printf("\n");
+  printf("Usage: 6pack [options]  input-file  output-file\n");
+  printf("\n");
+  printf("Options:\n");
+  printf("  -1    compress faster\n");
+  printf("  -2    compress better\n");
+  printf("  -v    show program version\n");
+#ifdef SIXPACK_BENCHMARK_WIN32
+  printf("  -mem  check in-memory compression speed\n");
+#endif
+  printf("\n");
+}
+
+/* return non-zero if magic sequence is detected */
+/* warning: reset the read pointer to the beginning of the file */
+int detect_magic(FILE *f)
+{
+  unsigned char buffer[8];
+  size_t bytes_read;
+  int c;
+
+  fseek(f, SEEK_SET, 0);
+  bytes_read = fread(buffer, 1, 8, f);
+  fseek(f, SEEK_SET, 0);
+  if(bytes_read < 8)
+    return 0;
+
+  for(c = 0; c < 8; c++)
+    if(buffer[c] != sixpack_magic[c])
+      return 0;
+
+  return -1;
+}
+
+void write_magic(FILE *f)
+{
+  fwrite(sixpack_magic, 8, 1, f);
+}
+
+void write_chunk_header(FILE* f, int id, int options, unsigned long size,
+  unsigned long checksum, unsigned long extra)
+{
+  unsigned char buffer[16];
+
+  buffer[0] = id & 255;
+  buffer[1] = id >> 8;
+  buffer[2] = options & 255;
+  buffer[3] = options >> 8;
+  buffer[4] = size & 255;
+  buffer[5] = (size >> 8) & 255;
+  buffer[6] = (size >> 16) & 255;
+  buffer[7] = (size >> 24) & 255;
+  buffer[8] = checksum & 255;
+  buffer[9] = (checksum >> 8) & 255;
+  buffer[10] = (checksum >> 16) & 255;
+  buffer[11] = (checksum >> 24) & 255;
+  buffer[12] = extra & 255;
+  buffer[13] = (extra >> 8) & 255;
+  buffer[14] = (extra >> 16) & 255;
+  buffer[15] = (extra >> 24) & 255;
+
+  fwrite(buffer, 16, 1, f);
+}
+
+int pack_file_compressed(const char* input_file, int method, int level, FILE* f)
+{
+  FILE* in;
+  unsigned long fsize;
+  unsigned long checksum;
+  const char* shown_name;
+  unsigned char buffer[BLOCK_SIZE];
+  unsigned char result[BLOCK_SIZE*2]; /* FIXME twice is too large */
+  unsigned char progress[20];
+  int c;
+  unsigned long percent;
+  unsigned long total_read;
+  unsigned long total_compressed;
+  int chunk_size;
+
+  /* sanity check */
+  in = fopen(input_file, "rb");
+  if(!in)
+  {
+    printf("Error: could not open %s\n", input_file);
+    return -1;
+  }
+
+  /* find size of the file */
+  fseek(in, 0, SEEK_END);
+  fsize = ftell(in);
+  fseek(in, 0, SEEK_SET);
+
+  /* already a 6pack archive? */
+  if(detect_magic(in))
+  {
+    printf("Error: file %s is already a 6pack archive!\n", input_file);
+    fclose(in);
+    return -1;
+  }
+
+  /* truncate directory prefix, e.g. "foo/bar/FILE.txt" becomes "FILE.txt" */
+  shown_name = input_file + strlen(input_file) - 1;
+  while(shown_name > input_file)
+    if(*(shown_name-1) == PATH_SEPARATOR)
+      break;
+    else
+      shown_name--;
+
+  /* chunk for File Entry */
+  buffer[0] = fsize & 255;
+  buffer[1] = (fsize >> 8) & 255;
+  buffer[2] = (fsize >> 16) & 255;
+  buffer[3] = (fsize >> 24) & 255;
+#if 0
+  buffer[4] = (fsize >> 32) & 255;
+  buffer[5] = (fsize >> 40) & 255;
+  buffer[6] = (fsize >> 48) & 255;
+  buffer[7] = (fsize >> 56) & 255;
+#else
+  /* because fsize is only 32-bit */
+  buffer[4] = 0;
+  buffer[5] = 0;
+  buffer[6] = 0;
+  buffer[7] = 0;
+#endif
+  buffer[8] = (strlen(shown_name)+1) & 255;
+  buffer[9] = (strlen(shown_name)+1) >> 8;
+  checksum = 1L;
+  checksum = update_adler32(checksum, buffer, 10);
+  checksum = update_adler32(checksum, shown_name, strlen(shown_name)+1);
+  write_chunk_header(f, 1, 0, 10+strlen(shown_name)+1, checksum, 0);
+  fwrite(buffer, 10, 1, f);
+  fwrite(shown_name, strlen(shown_name)+1, 1, f);
+  total_compressed = 16 + 10 + strlen(shown_name)+1;
+
+  /* for progress status */
+  memset(progress, ' ', 20);
+  if(strlen(shown_name) < 16)
+    for(c = 0; c < (int)strlen(shown_name); c++)
+      progress[c] = shown_name[c];
+  else
+  {
+    for(c = 0; c < 13; c++)
+      progress[c] = shown_name[c];
+    progress[13] = '.';
+    progress[14] = '.';
+    progress[15] = ' ';
+  }
+  progress[16] = '[';
+  progress[17] = 0;
+  printf("%s", progress);
+  for(c = 0; c < 50; c++)
+    printf(".");
+  printf("]\r");
+  printf("%s", progress);
+
+  /* read file and place in archive */
+  total_read = 0;
+  percent = 0;
+  for(;;)
+  {
+    int compress_method = method;
+    int last_percent = (int)percent;
+    size_t bytes_read = fread(buffer, 1, BLOCK_SIZE, in);
+    if(bytes_read == 0)
+      break;
+    total_read += bytes_read;
+
+    /* for progress */
+    if(fsize < (1<<24))
+      percent = total_read * 100 / fsize;
+    else
+      percent = total_read/256 * 100 / (fsize >>8);
+    percent >>= 1;
+    while(last_percent < (int)percent)
+    {
+      printf("#");
+      last_percent++;
+    }
+
+    /* too small, don't bother to compress */
+    if(bytes_read < 32)
+      compress_method = 0;
+
+    /* write to output */
+    switch(compress_method)
+    {
+      /* FastLZ */
+      case 1:
+        chunk_size = fastlz_compress_level(level, buffer, bytes_read, result);
+        checksum = update_adler32(1L, result, chunk_size);
+        write_chunk_header(f, 17, 1, chunk_size, checksum, bytes_read);
+        fwrite(result, 1, chunk_size, f);
+        total_compressed += 16;
+        total_compressed += chunk_size;
+        break;
+
+      /* uncompressed, also fallback method */
+      case 0:
+      default:
+        checksum = 1L;
+        checksum = update_adler32(checksum, buffer, bytes_read);
+        write_chunk_header(f, 17, 0, bytes_read, checksum, bytes_read);
+        fwrite(buffer, 1, bytes_read, f);
+        total_compressed += 16;
+        total_compressed += bytes_read;
+        break;
+    }
+  }
+
+  fclose(in);
+  if(total_read != fsize)
+  {
+    printf("\n");
+    printf("Error: reading %s failed!\n", input_file);
+    return -1;
+  }
+  else
+  {
+    printf("] ");
+    if(total_compressed < fsize)
+    {
+      if(fsize < (1<<20))
+        percent = total_compressed * 1000 / fsize;
+      else
+        percent = total_compressed/256 * 1000 / (fsize >>8);
+      percent = 1000 - percent;
+      printf("%2d.%d%% saved", (int)percent/10, (int)percent%10);
+    }
+    printf("\n");
+  }
+
+  return 0;
+}
+
+int pack_file(int compress_level, const char* input_file, const char* output_file)
+{
+  FILE* f;
+  int result;
+
+  f = fopen(output_file, "rb");
+  if(f)
+  {
+    fclose(f);
+    printf("Error: file %s already exists. Aborted.\n\n", output_file);
+    return -1;
+  }
+
+  f = fopen(output_file, "wb");
+  if(!f)
+  {
+    printf("Error: could not create %s. Aborted.\n\n", output_file);
+    return -1;
+  }
+
+  write_magic(f);
+
+  result = pack_file_compressed(input_file, 1, compress_level, f);
+  fclose(f);
+
+  return result;
+}
+
+#ifdef SIXPACK_BENCHMARK_WIN32
+int benchmark_speed(int compress_level, const char* input_file);
+
+int benchmark_speed(int compress_level, const char* input_file)
+{
+  FILE* in;
+  unsigned long fsize;
+  unsigned long maxout;
+  const char* shown_name;
+  unsigned char* buffer;
+  unsigned char* result;
+  size_t bytes_read;
+
+  /* sanity check */
+  in = fopen(input_file, "rb");
+  if(!in)
+  {
+    printf("Error: could not open %s\n", input_file);
+    return -1;
+  }
+
+  /* find size of the file */
+  fseek(in, 0, SEEK_END);
+  fsize = ftell(in);
+  fseek(in, 0, SEEK_SET);
+
+  /* already a 6pack archive? */
+  if(detect_magic(in))
+  {
+    printf("Error: no benchmark for 6pack archive!\n");
+    fclose(in);
+    return -1;
+  }
+
+  /* truncate directory prefix, e.g. "foo/bar/FILE.txt" becomes "FILE.txt" */
+  shown_name = input_file + strlen(input_file) - 1;
+  while(shown_name > input_file)
+    if(*(shown_name-1) == PATH_SEPARATOR)
+      break;
+    else
+      shown_name--;
+
+  maxout = 1.05 * fsize;
+  maxout = (maxout < 66) ? 66 : maxout;
+  buffer = (unsigned char*)malloc(fsize);
+  result = (unsigned char*)malloc(maxout);
+  if(!buffer || !result)
+  {
+    printf("Error: not enough memory!\n");
+    free(buffer);
+    free(result);
+    fclose(in);
+    return -1;
+  }
+
+  printf("Reading source file....\n");
+  bytes_read = fread(buffer, 1, fsize, in);
+  if(bytes_read != fsize)
+  {
+    printf("Error reading file %s!\n", shown_name);
+    printf("Read %d bytes, expecting %d bytes\n", bytes_read, fsize);
+    free(buffer);
+    free(result);
+    fclose(in);
+    return -1;
+  }
+
+/* shamelessly copied from QuickLZ 1.20 test program */
+  {
+    unsigned int j, y;
+    size_t i, u = 0;
+    double mbs, fastest;
+    unsigned long compressed_size;
+
+    printf("Setting HIGH_PRIORITY_CLASS...\n");
+    SetPriorityClass(GetCurrentProcess(), HIGH_PRIORITY_CLASS);
+
+    printf("Benchmarking FastLZ Level %d, please wait...\n", compress_level);
+
+    i = bytes_read;
+    fastest = 0.0;
+    for (j = 0; j < 3; j++)
+    {
+      y = 0;
+      mbs = GetTickCount();
+      while(GetTickCount() == mbs);
+      mbs = GetTickCount();
+      while(GetTickCount() - mbs < 3000) /* 1% accuracy with 18.2 timer */
+      {
+        u = fastlz_compress_level(compress_level, buffer, bytes_read, result);
+        y++;
+      }
+
+      mbs = ((double)i*(double)y)/((double)(GetTickCount() - mbs)/1000.)/1000000.;
+      /*printf(" %.1f Mbyte/s  ", mbs);*/
+      if (fastest	< mbs)
+        fastest = mbs;
+    }
+
+    printf("\nCompressed %d bytes into %d bytes (%.1f%%) at %.1f Mbyte/s.\n", (unsigned int)i, (unsigned int)u, (double)u/(double)i*100., fastest);
+
+#if 1
+    fastest = 0.0;
+    compressed_size = u;
+    for (j = 0; j < 3; j++)
+    {
+      y = 0;
+      mbs = GetTickCount();
+      while(GetTickCount() == mbs);
+      mbs = GetTickCount();
+      while(GetTickCount() - mbs < 3000) /* 1% accuracy with 18.2 timer */
+      {
+        u = fastlz_decompress(result, compressed_size, buffer, bytes_read);
+        y++;
+      }
+
+      mbs = ((double)i*(double)y)/((double)(GetTickCount() - mbs)/1000.)/1000000.;
+      /*printf(" %.1f Mbyte/s  ", mbs);*/
+      if (fastest	< mbs)
+        fastest = mbs;
+    }
+
+  printf("\nDecompressed at %.1f Mbyte/s.\n\n(1 MB = 1000000 byte)\n", fastest);
+#endif
+  }
+
+  fclose(in);
+  return 0;
+}
+#endif /* SIXPACK_BENCHMARK_WIN32 */
+
+
+int main(int argc, char** argv)
+{
+  int i;
+  int compress_level;
+  int benchmark;
+  char* input_file;
+  char* output_file;
+
+  /* show help with no argument at all*/
+  if(argc == 1)
+  {
+    usage();
+    return 0;
+  }
+
+  /* default compression level, not the fastest */
+  compress_level = 2;
+
+  /* do benchmark only when explicitly specified */
+  benchmark = 0;
+
+  /* no file is specified */
+  input_file = 0;
+  output_file = 0;
+
+  for(i = 1; i <= argc; i++)
+  {
+    char* argument = argv[i];
+
+    if(!argument)
+      continue;
+
+    /* display help on usage */
+    if(!strcmp(argument, "-h") || !strcmp(argument, "--help"))
+    {
+      usage();
+      return 0;
+    }
+
+    /* check for version information */
+    if(!strcmp(argument, "-v") || !strcmp(argument, "--version"))
+    {
+      printf("6pack: high-speed file compression tool\n");
+      printf("Version %s (using FastLZ %s)\n",
+        SIXPACK_VERSION_STRING, FASTLZ_VERSION_STRING);
+      printf("Copyright (C) 2007 Ariya Hidayat (ariya@kde.org)\n");
+      printf("\n");
+      return 0;
+    }
+
+    /* test compression speed? */
+    if(!strcmp(argument, "-mem"))
+    {
+      benchmark = 1;
+      continue;
+    }
+
+    /* compression level */
+    if(!strcmp(argument, "-1") || !strcmp(argument, "--fastest"))
+    {
+      compress_level = 1;
+      continue;
+    }
+    if(!strcmp(argument, "-2"))
+    {
+      compress_level = 2;
+      continue;
+    }
+
+    /* unknown option */
+    if(argument[0] == '-')
+    {
+      printf("Error: unknown option %s\n\n", argument);
+      printf("To get help on usage:\n");
+      printf("  6pack --help\n\n");
+      return -1;
+    }
+
+    /* first specified file is input */
+    if(!input_file)
+    {
+      input_file = argument;
+      continue;
+    }
+
+    /* next specified file is output */
+    if(!output_file)
+    {
+      output_file = argument;
+      continue;
+    }
+
+    /* files are already specified */
+    printf("Error: unknown option %s\n\n", argument);
+    printf("To get help on usage:\n");
+    printf("  6pack --help\n\n");
+    return -1;
+  }
+
+  if(!input_file)
+  {
+    printf("Error: input file is not specified.\n\n");
+    printf("To get help on usage:\n");
+    printf("  6pack --help\n\n");
+    return -1;
+  }
+
+  if(!output_file && !benchmark)
+  {
+    printf("Error: output file is not specified.\n\n");
+    printf("To get help on usage:\n");
+    printf("  6pack --help\n\n");
+    return -1;
+  }
+
+#ifdef SIXPACK_BENCHMARK_WIN32
+  if(benchmark)
+    return benchmark_speed(compress_level, input_file);
+  else
+#endif
+    return pack_file(compress_level, input_file, output_file);
+
+  /* unreachable */
+  return 0;
+}

+ 478 - 0
poc/fastlz/6unpack.c

@@ -0,0 +1,478 @@
+/*
+  6PACK - file compressor using FastLZ (lightning-fast compression library)
+
+  Copyright (C) 2007 Ariya Hidayat (ariya@kde.org)
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define SIXPACK_VERSION_MAJOR    0
+#define SIXPACK_VERSION_MINOR    1
+#define SIXPACK_VERSION_REVISION 0
+#define SIXPACK_VERSION_STRING   "0.1.0"
+
+#include "fastlz.h"
+
+#if defined(WIN32) || defined(__NT__) || defined(_WIN32) || defined(__WIN32__)
+#if defined(__BORLANDC__) || defined(_MSC_VER)
+#define inline __inline
+#endif
+#endif
+
+/* magic identifier for 6pack file */
+static unsigned char sixpack_magic[8] = {137, '6', 'P', 'K', 13, 10, 26, 10};
+
+#define BLOCK_SIZE 65536
+
+/* prototypes */
+static inline unsigned long update_adler32(unsigned long checksum, const void *buf, int len);
+void usage(void);
+int detect_magic(FILE *f);
+static inline unsigned long readU16(const unsigned char* ptr);
+static inline unsigned long readU32(const unsigned char* ptr);
+void read_chunk_header(FILE* f, int* id, int* options, unsigned long* size,
+unsigned long* checksum, unsigned long* extra);
+int unpack_file(const char* archive_file);
+
+/* for Adler-32 checksum algorithm, see RFC 1950 Section 8.2 */
+#define ADLER32_BASE 65521
+static inline unsigned long update_adler32(unsigned long checksum, const void *buf, int len)
+{
+  const unsigned char* ptr = (const unsigned char*)buf;
+  unsigned long s1 = checksum & 0xffff;
+  unsigned long s2 = (checksum >> 16) & 0xffff;
+
+  while(len>0)
+  {
+    unsigned k = len < 5552 ? len : 5552;
+    len -= k;
+
+    while(k >= 8)
+    {
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      s1 += *ptr++; s2 += s1;
+      k -= 8;
+    }
+
+    while(k-- > 0)
+    {
+      s1 += *ptr++; s2 += s1;
+    }
+    s1 = s1 % ADLER32_BASE;
+    s2 = s2 % ADLER32_BASE;
+  }
+  return (s2 << 16) + s1;
+}
+
+void usage(void)
+{
+  printf("6unpack: uncompress 6pack archive\n");
+  printf("Copyright (C) 2007 Ariya Hidayat (ariya@kde.org)\n");
+  printf("\n");
+  printf("Usage: 6unpack archive-file\n");
+  printf("\n");
+}
+
+/* return non-zero if magic sequence is detected */
+/* warning: reset the read pointer to the beginning of the file */
+int detect_magic(FILE *f)
+{
+  unsigned char buffer[8];
+  size_t bytes_read;
+  int c;
+
+  fseek(f, SEEK_SET, 0);
+  bytes_read = fread(buffer, 1, 8, f);
+  fseek(f, SEEK_SET, 0);
+  if(bytes_read < 8)
+    return 0;
+
+  for(c = 0; c < 8; c++)
+    if(buffer[c] != sixpack_magic[c])
+      return 0;
+
+  return -1;
+}
+
+static inline unsigned long readU16( const unsigned char* ptr )
+{
+  return ptr[0]+(ptr[1]<<8);
+}
+
+static inline unsigned long readU32( const unsigned char* ptr )
+{
+  return ptr[0]+(ptr[1]<<8)+(ptr[2]<<16)+(ptr[3]<<24);
+}
+
+void read_chunk_header(FILE* f, int* id, int* options, unsigned long* size,
+unsigned long* checksum, unsigned long* extra)
+{
+  unsigned char buffer[16];
+  fread(buffer, 1, 16, f);
+
+  *id = readU16(buffer) & 0xffff;
+  *options = readU16(buffer+2) & 0xffff;
+  *size = readU32(buffer+4) & 0xffffffff;
+  *checksum = readU32(buffer+8) & 0xffffffff;
+  *extra = readU32(buffer+12) & 0xffffffff;
+}
+
+int unpack_file(const char* input_file)
+{
+  FILE* in;
+  unsigned long fsize;
+  int c;
+  unsigned long percent;
+  unsigned char progress[20];
+  int chunk_id;
+  int chunk_options;
+  unsigned long chunk_size;
+  unsigned long chunk_checksum;
+  unsigned long chunk_extra;
+  unsigned char buffer[BLOCK_SIZE];
+  unsigned long checksum;
+
+  unsigned long decompressed_size;
+  unsigned long total_extracted;
+  int name_length;
+  char* output_file;
+  FILE* f;
+
+  unsigned char* compressed_buffer;
+  unsigned char* decompressed_buffer;
+  unsigned long compressed_bufsize;
+  unsigned long decompressed_bufsize;
+
+  /* sanity check */
+  in = fopen(input_file, "rb");
+  if(!in)
+  {
+    printf("Error: could not open %s\n", input_file);
+    return -1;
+  }
+
+  /* find size of the file */
+  fseek(in, 0, SEEK_END);
+  fsize = ftell(in);
+  fseek(in, 0, SEEK_SET);
+
+  /* not a 6pack archive? */
+  if(!detect_magic(in))
+  {
+    fclose(in);
+    printf("Error: file %s is not a 6pack archive!\n", input_file);
+    return -1;
+  }
+
+  printf("Archive: %s", input_file);
+
+  /* position of first chunk */
+  fseek(in, 8, SEEK_SET);
+
+  /* initialize */
+  output_file = 0;
+  f = 0;
+  total_extracted = 0;
+  decompressed_size = 0;
+  percent = 0;
+  compressed_buffer = 0;
+  decompressed_buffer = 0;
+  compressed_bufsize = 0;
+  decompressed_bufsize = 0;
+
+  /* main loop */
+  for(;;)
+  {
+    /* end of file? */
+    size_t pos = ftell(in);
+    if(pos >= fsize)
+      break;
+
+    read_chunk_header(in, &chunk_id, &chunk_options,
+        &chunk_size, &chunk_checksum, &chunk_extra);
+
+    if((chunk_id == 1) && (chunk_size > 10) && (chunk_size < BLOCK_SIZE))
+    {
+      /* close current file, if any */
+      printf("\n");
+      free(output_file);
+      output_file = 0;
+      if(f)
+        fclose(f);
+
+      /* file entry */
+      fread(buffer, 1, chunk_size, in);
+      checksum = update_adler32(1L, buffer, chunk_size);
+      if(checksum != chunk_checksum)
+      {
+        free(output_file);
+        output_file = 0;
+        fclose(in);
+        printf("\nError: checksum mismatch!\n");
+        printf("Got %08lX Expecting %08lX\n", checksum, chunk_checksum);
+        return -1;
+      }
+
+      decompressed_size = readU32(buffer);
+      total_extracted = 0;
+      percent = 0;
+
+      /* get file to extract */
+      name_length = (int)readU16(buffer+8);
+      if(name_length > (int)chunk_size - 10)
+        name_length = chunk_size - 10;
+      output_file = (char*)malloc(name_length+1);
+      memset(output_file, 0, name_length+1);
+      for(c = 0; c < name_length; c++)
+        output_file[c] = buffer[10+c];
+
+      /* check if already exists */
+      f = fopen(output_file, "rb");
+      if(f)
+      {
+        fclose(f);
+        printf("File %s already exists. Skipped.\n", output_file);
+        free(output_file);
+        output_file = 0;
+        f = 0;
+      }
+      else
+      {
+        /* create the file */
+        f = fopen(output_file, "wb");
+        if(!f)
+        {
+          printf("Can't create file %s. Skipped.\n", output_file);
+          free(output_file);
+          output_file = 0;
+          f = 0;
+        }
+        else
+        {
+          /* for progress status */
+          printf("\n");
+          memset(progress, ' ', 20);
+          if(strlen(output_file) < 16)
+            for(c = 0; c < (int)strlen(output_file); c++)
+              progress[c] = output_file[c];
+          else
+          {
+            for(c = 0; c < 13; c++)
+              progress[c] = output_file[c];
+            progress[13] = '.';
+            progress[14] = '.';
+            progress[15] = ' ';
+          }
+          progress[16] = '[';
+          progress[17] = 0;
+          printf("%s", progress);
+          for(c = 0; c < 50; c++)
+            printf(".");
+          printf("]\r");
+          printf("%s", progress);
+        }
+      }
+    }
+
+    if((chunk_id == 17) && f && output_file && decompressed_size)
+    {
+      unsigned long remaining;
+
+      /* uncompressed */
+      switch(chunk_options)
+      {
+        /* stored, simply copy to output */
+        case 0:
+          /* read one block at at time, write and update checksum */
+          total_extracted += chunk_size;
+          remaining = chunk_size;
+          checksum = 1L;
+          for(;;)
+          {
+            unsigned long r = (BLOCK_SIZE < remaining) ? BLOCK_SIZE: remaining;
+            size_t bytes_read = fread(buffer, 1, r, in);
+            if(bytes_read == 0)
+              break;
+            fwrite(buffer, 1, bytes_read, f);
+            checksum = update_adler32(checksum, buffer, bytes_read);
+            remaining -= bytes_read;
+          }
+
+          /* verify everything is written correctly */
+          if(checksum != chunk_checksum)
+          {
+            fclose(f);
+            f = 0;
+            free(output_file);
+            output_file = 0;
+            printf("\nError: checksum mismatch. Aborted.\n");
+            printf("Got %08lX Expecting %08lX\n", checksum, chunk_checksum);
+          }
+          break;
+
+        /* compressed using FastLZ */
+        case 1:
+          /* enlarge input buffer if necessary */
+          if(chunk_size > compressed_bufsize)
+          {
+            compressed_bufsize = chunk_size;
+            free(compressed_buffer);
+            compressed_buffer = (unsigned char*)malloc(compressed_bufsize);
+          }
+
+          /* enlarge output buffer if necessary */
+          if(chunk_extra > decompressed_bufsize)
+          {
+            decompressed_bufsize = chunk_extra;
+            free(decompressed_buffer);
+            decompressed_buffer = (unsigned char*)malloc(decompressed_bufsize);
+          }
+
+          /* read and check checksum */
+          fread(compressed_buffer, 1, chunk_size, in);
+          checksum = update_adler32(1L, compressed_buffer, chunk_size);
+          total_extracted += chunk_extra;
+
+          /* verify that the chunk data is correct */
+          if(checksum != chunk_checksum)
+          {
+            fclose(f);
+            f = 0;
+            free(output_file);
+            output_file = 0;
+            printf("\nError: checksum mismatch. Skipped.\n");
+            printf("Got %08lX Expecting %08lX\n", checksum, chunk_checksum);
+          }
+          else
+          {
+            /* decompress and verify */
+            remaining = fastlz_decompress(compressed_buffer, chunk_size, decompressed_buffer, chunk_extra);
+            if(remaining != chunk_extra)
+            {
+              fclose(f);
+              f = 0;
+              free(output_file);
+              output_file = 0;
+              printf("\nError: decompression failed. Skipped.\n");
+            }
+            else
+              fwrite(decompressed_buffer, 1, chunk_extra, f);
+          }
+          break;
+
+        default:
+          printf("\nError: unknown compression method (%d)\n", chunk_options);
+          fclose(f);
+          f = 0;
+          free(output_file);
+          output_file = 0;
+          break;
+      }
+
+      /* for progress, if everything is fine */
+      if(f)
+      {
+        int last_percent = (int)percent;
+        if(decompressed_size < (1<<24))
+          percent = total_extracted * 100 / decompressed_size;
+        else
+          percent = total_extracted / 256 * 100 / (decompressed_size >>8);
+        percent >>= 1;
+        while(last_percent < (int)percent)
+        {
+          printf("#");
+          last_percent++;
+        }
+      }
+    }
+
+    /* position of next chunk */
+    fseek(in, pos + 16 + chunk_size, SEEK_SET);
+  }
+  printf("\n\n");
+
+  /* free allocated stuff */
+  free(compressed_buffer);
+  free(decompressed_buffer);
+  free(output_file);
+
+  /* close working files */
+  if(f)
+    fclose(f);
+  fclose(in);
+
+  /* so far so good */
+  return 0;
+}
+
+int main(int argc, char** argv)
+{
+  int i;
+  const char* archive_file;
+
+  /* show help with no argument at all*/
+  if(argc == 1)
+  {
+    usage();
+    return 0;
+  }
+
+  /* check for help on usage */
+  for(i = 1; i <= argc; i++)
+    if(argv[i])
+    if(!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help"))
+    {
+      usage();
+      return 0;
+    }
+
+  /* check for version information */
+  for(i = 1; i <= argc; i++)
+    if(argv[i])
+    if(!strcmp(argv[i], "-v") || !strcmp(argv[i], "--version"))
+    {
+      printf("6unpack: high-speed file compression tool\n");
+      printf("Version %s (using FastLZ %s)\n",
+        SIXPACK_VERSION_STRING, FASTLZ_VERSION_STRING);
+      printf("Copyright (C) 2007 Ariya Hidayat (ariya@kde.org)\n");
+      printf("\n");
+      return 0;
+    }
+
+  /* needs at least two arguments */
+  if(argc <= 1)
+  {
+    usage();
+    return 0;
+  }
+
+  archive_file = argv[1];
+
+  return unpack_file(archive_file);
+}

+ 5 - 0
poc/fastlz/Makefile

@@ -0,0 +1,5 @@
+
+
+all:
+	gcc -o poc poc.c fastlz.c -I/usr/local/Cellar/openssl/1.0.2g/include -L/usr/local/Cellar/openssl/1.0.2g/lib -lssl  -lcrypto 
+

+ 426 - 0
poc/fastlz/fastlz.c

@@ -0,0 +1,426 @@
+/*  
+  FastLZ - lightning-fast lossless compression library
+
+  Copyright (C) 2007 Ariya Hidayat (ariya@kde.org)
+  Copyright (C) 2006 Ariya Hidayat (ariya@kde.org)
+  Copyright (C) 2005 Ariya Hidayat (ariya@kde.org)
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
+*/
+
+/*
+ * Give hints to the compiler for branch prediction optimization.
+ */
+#if defined(__GNUC__) && (__GNUC__ > 2)
+#define FASTLZ_EXPECT_CONDITIONAL(c)    (__builtin_expect((c), 1))
+#define FASTLZ_UNEXPECT_CONDITIONAL(c)  (__builtin_expect((c), 0))
+#else
+#define FASTLZ_EXPECT_CONDITIONAL(c)    (c)
+#define FASTLZ_UNEXPECT_CONDITIONAL(c)  (c)
+#endif
+
+/*
+ * Use inlined functions for supported systems.
+ */
+#if defined(__GNUC__) || defined(__DMC__) || defined(__POCC__) || defined(__WATCOMC__) || defined(__SUNPRO_C)
+#define FASTLZ_INLINE inline
+#elif defined(__BORLANDC__) || defined(_MSC_VER) || defined(__LCC__)
+#define FASTLZ_INLINE __inline
+#else 
+#define FASTLZ_INLINE
+#endif
+
+typedef unsigned char  flzuint8;
+typedef unsigned short flzuint16;
+typedef unsigned int   flzuint32;
+
+/* prototypes */
+int fastlz_compress(const void* input, int length, void* output);
+int fastlz_decompress(const void* input, int length, void* output);
+
+#define MAX_COPY       32
+#define MAX_LEN       264  /* 256 + 8 */
+#define MAX_DISTANCE  256
+
+#define FASTLZ_READU16(p) ((p)[0] | (p)[1]<<8)
+
+#define HASH_LOG  13
+#define HASH_SIZE (1<< HASH_LOG)
+#define HASH_MASK  (HASH_SIZE-1)
+#define HASH_FUNCTION(v,p) { v = FASTLZ_READU16(p); v ^= FASTLZ_READU16(p+1)^(v>>(16-HASH_LOG));v &= HASH_MASK; }
+
+
+FASTLZ_INLINE int fastlz_compress(const void* input, int length, void* output)
+{
+  const flzuint8* ip = (const flzuint8*) input;
+  const flzuint8* ip_bound = ip + length - 2;
+  const flzuint8* ip_limit = ip + length - 12;
+  flzuint8* op = (flzuint8*) output;
+
+  const flzuint8* htab[HASH_SIZE];
+  const flzuint8** hslot;
+  flzuint32 hval;
+
+  flzuint32 copy;
+
+  /* sanity check */
+  if(FASTLZ_UNEXPECT_CONDITIONAL(length < 4))
+  {
+    if(length)
+    {
+      /* create literal copy only */
+      *op++ = length-1;
+      ip_bound++;
+      while(ip <= ip_bound)
+        *op++ = *ip++;
+      return length+1;
+    }
+    else
+      return 0;
+  }
+
+  /* initializes hash table */
+  for (hslot = htab; hslot < htab + HASH_SIZE; hslot++)
+    *hslot = ip;
+
+  /* we start with literal copy */
+  copy = 2;
+  *op++ = MAX_COPY-1;
+  *op++ = *ip++;
+  *op++ = *ip++;
+
+  /* main loop */
+  while(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit))
+  {
+    const flzuint8* ref;
+    flzuint32 distance;
+
+    /* minimum match length */
+    flzuint32 len = 3;
+
+    /* comparison starting-point */
+    const flzuint8* anchor = ip;
+
+    /* check for a run */
+
+    /* find potential match */
+    HASH_FUNCTION(hval,ip);
+    hslot = htab + hval;
+    ref = htab[hval];
+
+    /* calculate distance to the match */
+    distance = anchor - ref;
+
+    /* update hash table */
+    *hslot = anchor;
+
+    /* is this a match? check the first 3 bytes */
+    if(distance==0 || 
+    (distance >= MAX_DISTANCE) ||
+    *ref++ != *ip++ || *ref++!=*ip++ || *ref++!=*ip++)
+      goto literal;
+
+
+    /* last matched byte */
+    ip = anchor + len;
+
+    /* distance is biased */
+    distance--;
+
+    if(!distance)
+    {
+      /* zero distance means a run */
+      flzuint8 x = ip[-1];
+      while(ip < ip_bound)
+        if(*ref++ != x) break; else ip++;
+    }
+    else
+    for(;;)
+    {
+      /* safe because the outer check against ip limit */
+      if(*ref++ != *ip++) break;
+      if(*ref++ != *ip++) break;
+      if(*ref++ != *ip++) break;
+      if(*ref++ != *ip++) break;
+      if(*ref++ != *ip++) break;
+      if(*ref++ != *ip++) break;
+      if(*ref++ != *ip++) break;
+      if(*ref++ != *ip++) break;
+      while(ip < ip_bound)
+        if(*ref++ != *ip++) break;
+      break;
+    }
+
+    /* if we have copied something, adjust the copy count */
+    if(copy)
+      /* copy is biased, '0' means 1 byte copy */
+      *(op-copy-1) = copy-1;
+    else
+      /* back, to overwrite the copy count */
+      op--;
+
+    /* reset literal counter */
+    copy = 0;
+
+    /* length is biased, '1' means a match of 3 bytes */
+    ip -= 3;
+    len = ip - anchor;
+
+    /* encode the match */
+    if(FASTLZ_UNEXPECT_CONDITIONAL(len > MAX_LEN-2))
+      while(len > MAX_LEN-2)
+      {
+        *op++ = (7 << 5) + (distance >> 8);
+        *op++ = MAX_LEN - 2 - 7 -2; 
+        *op++ = (distance & 255);
+        len -= MAX_LEN-2;
+      }
+
+    if(len < 7)
+    {
+      *op++ = (len << 5) + (distance >> 8);
+      *op++ = (distance & 255);
+    }
+    else
+    {
+      *op++ = (7 << 5) + (distance >> 8);
+      *op++ = len - 7;
+      *op++ = (distance & 255);
+    }
+
+    /* update the hash at match boundary */
+    HASH_FUNCTION(hval,ip);
+    htab[hval] = ip++;
+    HASH_FUNCTION(hval,ip);
+    htab[hval] = ip++;
+
+    /* assuming literal copy */
+    *op++ = MAX_COPY-1;
+
+    continue;
+
+    literal:
+      *op++ = *anchor++;
+      ip = anchor;
+      copy++;
+      if(FASTLZ_UNEXPECT_CONDITIONAL(copy == MAX_COPY))
+      {
+        copy = 0;
+        *op++ = MAX_COPY-1;
+      }
+  }
+
+  /* left-over as literal copy */
+  ip_bound++;
+  while(ip <= ip_bound)
+  {
+    *op++ = *ip++;
+    copy++;
+    if(copy == MAX_COPY)
+    {
+      copy = 0;
+      *op++ = MAX_COPY-1;
+    }
+  }
+
+  /* if we have copied something, adjust the copy length */
+  if(copy)
+    *(op-copy-1) = copy-1;
+  else
+    op--;
+
+  return op - (flzuint8*)output;
+}
+
+
+#include <stdio.h>
+#define log1(NUM) printf("%i op=%i(%x) ip=%i(%x) len=%i ctrl=%i ofs=%i(%i) limit=%i\n",NUM,  (((int)op - (int)output)), *op, (((int)ip - (int)input)),*ip,  len, ctrl, ofs, (ofs >> 8),ip < ip_limit);
+
+int fastlz_decompress(const void* input, int length, void* output)
+{
+  const flzuint8* ip = (const flzuint8*) input;
+  const flzuint8* ip_limit  = ip + length;
+  flzuint8* op = (flzuint8*) output;
+  flzuint32 ctrl = (*ip++) & 31;
+  int loop = 1;
+  do
+  {
+    const flzuint8* ref = op;
+    flzuint32 len = ctrl >> 5;
+    flzuint32 ofs = (ctrl & 31) << 8;
+    printf("-------------------\n");
+    log1(1)
+    if(ctrl >= 32)
+    {
+      len--;
+      ref -= ofs;
+      if (len == 7-1)
+        len += *ip++;
+      ref -= *ip++;
+      
+
+      log1(1)
+      if(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit))
+        ctrl = *ip++;
+      else
+        loop = 0;
+      log1(1)
+
+      if(ref == op)
+      {
+        log1(2)
+        /* optimize copy for a run */
+        flzuint8 b = ref[-1];
+        *op++ = b;
+        *op++ = b;
+        *op++ = b;
+        for(; len; --len)
+          *op++ = b;
+      }
+      else
+      {
+        log1(3)
+        /* copy from reference */
+        ref--;
+        *op++ = *ref++;
+        *op++ = *ref++;
+        *op++ = *ref++;
+
+        for(; len; --len)
+          *op++ = *ref++;
+      }
+    }
+    else
+    {
+      ctrl++;
+      log1(4)
+      *op++ = *ip++; 
+      for(--ctrl; ctrl; ctrl--){
+        log1(5)
+        *op++ = *ip++;
+     }
+
+      loop = FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit);
+      if(loop){
+        ctrl = *ip++;
+      }
+      log1(6)
+    }
+  }
+  while(FASTLZ_EXPECT_CONDITIONAL(loop));
+
+  return op - (flzuint8*)output;
+
+}
+#include <stdlib.h>
+#include "ringbuffer.h"
+
+#define log2(NUM) printf("%i op=%i(%x) ip=%i(%x) ref=%i(%x) dist=%i len=%i ctrl=%i ofs=%i(%i) limit=%i\n",NUM, output_addr,output[output_addr], input_addr,input[input_addr],ref_addr,input[ref_addr],output_addr - ref_addr, len, ctrl, ofs, ofs>>6,input_addr < ip_limit); 
+
+#define OUTPUT_INC(B)  do { \
+    flzuint8 __b = B;\
+    output[output_addr] = __b;\
+    bufferWrite(buffer_ptr, __b);\
+    output_addr++;\
+} while (0)
+
+
+
+ringBuffer_typedef(unsigned char, charBuffer);
+
+int fastlz_decompress2(unsigned char* input, int length, unsigned char* output)
+{
+  flzuint32 input_addr = 0;
+  flzuint32 ip_limit = length;
+  flzuint32 output_addr = 0;
+  flzuint32 ref_addr = 0;
+  flzuint32 ctrl = (input[input_addr++]) & 31;
+  int loop = 1;
+
+  charBuffer buffer;
+  bufferInit(buffer, MAX_DISTANCE, unsigned char);
+  charBuffer* buffer_ptr;
+  buffer_ptr = &buffer;
+
+  do
+  {
+    ref_addr = output_addr;
+    flzuint32 len = ctrl >> 5;
+    flzuint32 ofs = (ctrl & 31) << 6;
+    printf("-------------------\n");
+    log2(1)
+    if(ctrl >= 32)
+    {
+      len--;
+      ref_addr -= ofs;
+      if (len == 7-1)
+        len += input[input_addr++];
+      ref_addr -= input[input_addr++];
+      
+      log2(1)
+
+      if(FASTLZ_EXPECT_CONDITIONAL( input_addr < ip_limit))
+        ctrl = input[input_addr++];
+      else
+        loop = 0;
+      
+      log2(1)
+
+      if(ref_addr == output_addr)
+      {
+        log2(2)
+        flzuint8 b = output[ref_addr-1];
+        OUTPUT_INC(b);
+        OUTPUT_INC(b);
+        OUTPUT_INC(b);
+        for(; len; --len)
+            OUTPUT_INC(b);
+      }
+      else
+      {
+        log2(3)
+        ref_addr--;
+        OUTPUT_INC(output[ref_addr++]);
+        OUTPUT_INC(output[ref_addr++]);
+        OUTPUT_INC(output[ref_addr++]);
+        for(; len; --len)
+          OUTPUT_INC(output[ref_addr++]);
+      }
+    }
+    else
+    {
+      ctrl++;
+      log2(4)
+      OUTPUT_INC(input[input_addr++]);
+      for(--ctrl; ctrl; ctrl--){
+        log2(5)
+        OUTPUT_INC(input[input_addr++]);
+    } 
+
+      loop = FASTLZ_EXPECT_CONDITIONAL(input_addr < ip_limit);
+      if (loop){
+        ctrl = input[input_addr++];
+     }
+      log2(6)
+    }
+  }
+  while(FASTLZ_EXPECT_CONDITIONAL(loop));
+
+  return 0;
+}

+ 99 - 0
poc/fastlz/fastlz.h

@@ -0,0 +1,99 @@
+/*  
+  FastLZ - lightning-fast lossless compression library
+
+  Copyright (C) 2007 Ariya Hidayat (ariya@kde.org)
+  Copyright (C) 2006 Ariya Hidayat (ariya@kde.org)
+  Copyright (C) 2005 Ariya Hidayat (ariya@kde.org)
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+  THE SOFTWARE.
+*/
+
+#ifndef FASTLZ_H
+#define FASTLZ_H
+
+#define FASTLZ_VERSION 0x000100
+
+#define FASTLZ_VERSION_MAJOR     0
+#define FASTLZ_VERSION_MINOR     0
+#define FASTLZ_VERSION_REVISION  0
+
+#define FASTLZ_VERSION_STRING "0.1.0"
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/**
+  Compress a block of data in the input buffer and returns the size of 
+  compressed block. The size of input buffer is specified by length. The 
+  minimum input buffer size is 16.
+
+  The output buffer must be at least 5% larger than the input buffer  
+  and can not be smaller than 66 bytes.
+
+  If the input is not compressible, the return value might be larger than
+  length (input buffer size).
+
+  The input buffer and the output buffer can not overlap.
+*/
+
+int fastlz_compress(const void* input, int length, void* output);
+
+/**
+  Decompress a block of compressed data and returns the size of the 
+  decompressed block. If error occurs, e.g. the compressed data is 
+  corrupted or the output buffer is not large enough, then 0 (zero) 
+  will be returned instead.
+
+  The input buffer and the output buffer can not overlap.
+
+  Decompression is memory safe and guaranteed not to write the output buffer
+  more than what is specified in maxout.
+ */
+
+int fastlz_decompress(const void* input, int length, void* output); 
+
+/**
+  Compress a block of data in the input buffer and returns the size of 
+  compressed block. The size of input buffer is specified by length. The 
+  minimum input buffer size is 16.
+
+  The output buffer must be at least 5% larger than the input buffer  
+  and can not be smaller than 66 bytes.
+
+  If the input is not compressible, the return value might be larger than
+  length (input buffer size).
+
+  The input buffer and the output buffer can not overlap.
+
+  Compression level can be specified in parameter level. At the moment, 
+  only level 1 and level 2 are supported.
+  Level 1 is the fastest compression and generally useful for short data.
+  Level 2 is slightly slower but it gives better compression ratio.
+
+  Note that the compressed data, regardless of the level, can always be
+  decompressed using the function fastlz_decompress above.
+*/  
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* FASTLZ_H */

+ 150 - 0
poc/fastlz/poc.c

@@ -0,0 +1,150 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include "fastlz.h"
+#include <openssl/md5.h>
+
+#define HEXDUMP_COLS 8
+
+void hexdump(void *mem, unsigned int len);
+
+unsigned char *unpacked;
+unsigned char *packed;
+
+void init(long len){
+    unpacked = (char*)malloc(len);
+    packed = (char*)malloc(len);
+
+    memset(unpacked, 0, len);
+    memset(packed, 0, len);
+}
+
+int main(int argc, char** argv){
+    
+    char *source = NULL;
+    int len, rlen, i;
+    FILE *fp = fopen("/Users/david/Devel/arch/avr/code/quickdev16/roms/qd16boot02.smc", "r");
+    if (fp != NULL) {
+        /* Go to the end of the file. */
+        if (fseek(fp, 0L, SEEK_END) == 0) {
+            /* Get the size of the file. */
+            len = ftell(fp);
+            if (len == -1) { /* Error */ }
+
+            /* Allocate our buffer to that size. */
+            source = malloc(sizeof(char) * (len + 1));
+
+            /* Go back to the start of the file. */
+            if (fseek(fp, 0L, SEEK_SET) != 0) { /* Error */ }
+
+            /* Read the entire file into memory. */
+            size_t newLen = fread(source, sizeof(char), len, fp);
+            if (newLen == 0) {
+                fputs("Error reading file", stderr);
+            } else {
+                printf("Reading file with size=%i\n", len);
+                source[newLen++] = '\0'; /* Just to be safe. */
+            }
+        }
+        fclose(fp);
+    }
+    char sample[] = "Lorem ipsum dolor sit amet, labore commodo platonem no vis, in ius atqui dicunt, omnium euismod nam in. Ei integre euismod philosophia mea. Qui fabulas comprehensam at, omnium corrumpit comprehensam mel no, sed audiam maiestatis et. Brute perfecto et his, mea cu inermis facilis scriptorem, eam te admodum urbanitas intellegam. At vis regione invidunt tractatos, ea modus errem has.";
+   
+
+    MD5_CTX md5_context;
+    unsigned char c[MD5_DIGEST_LENGTH];
+
+#if 0
+    len = strlen(sample);
+    init(len);
+    memcpy(unpacked, sample, len);
+#else 
+    init(len);
+    memcpy(unpacked, source, len);
+#endif
+
+    MD5_Init (&md5_context); 
+    MD5_Update (&md5_context, unpacked, len);
+    MD5_Final (c,&md5_context);
+    printf("unpacked len=%i md5=", len);
+    for(i = 0; i < MD5_DIGEST_LENGTH; i++) printf("%02x", c[i]);
+    printf("\n");
+    rlen = fastlz_compress(unpacked, len, packed);
+    hexdump(packed, rlen);
+    printf("packed len=%i\n", rlen);
+    memset(unpacked, 0, len);
+    printf("-----\n");
+    fastlz_decompress(packed, rlen, unpacked);
+    MD5_Init (&md5_context); 
+    MD5_Update (&md5_context, unpacked, len);
+    MD5_Final (c,&md5_context);
+    printf("unpacked len=%i md5=", len);
+    for(i = 0; i < MD5_DIGEST_LENGTH; i++) printf("%02x", c[i]);
+    printf("\n");
+    fp = fopen("out01.smc", "wb");
+    fwrite(unpacked, len, 1, fp);
+    printf("Wrote out01.smc %l bytes\n", len);
+    fclose(fp);
+    memset(unpacked, 0, len);
+    printf("-----\n");
+    fastlz_decompress2(packed, rlen, unpacked);
+    MD5_Init (&md5_context); 
+    MD5_Update (&md5_context, unpacked, len);
+    MD5_Final (c,&md5_context);
+    printf("unpacked len=%i md5=", len);
+    for(i = 0; i < MD5_DIGEST_LENGTH; i++) printf("%02x", c[i]);
+    printf("\n");
+    fp = fopen("out02.smc", "wb");
+    fwrite(unpacked, len, 1, fp);
+    printf("Wrote out02.smc %l bytes\n", len);
+    fclose(fp);
+    printf("s1=%s\n\n", sample);
+    printf("s2=%s\n", unpacked);
+
+
+    return 0;
+}
+
+void hexdump(void *mem, unsigned int len){
+    unsigned int i, j;
+    for(i = 0; i < len + ((len % HEXDUMP_COLS) ? (HEXDUMP_COLS - len % HEXDUMP_COLS) : 0); i++)
+    {
+        /* print offset */
+        if(i % HEXDUMP_COLS == 0)
+        {
+                printf("0x%06x: ", i);
+        }
+
+        /* print hex data */
+        if(i < len)
+        {
+                printf("%02x ", 0xFF & ((char*)mem)[i]);
+        }
+        else /* end of block, just aligning for ASCII dump */
+        {
+                printf("   ");
+        }
+        
+        /* print ASCII dump */
+        if(i % HEXDUMP_COLS == (HEXDUMP_COLS - 1))
+        {
+            for(j = i - (HEXDUMP_COLS - 1); j <= i; j++)
+            {
+                    if(j >= len) /* end of block, not really printing */
+                    {
+                            putchar(' ');
+                    }
+                    else if(isprint(((char*)mem)[j])) /* printable char */
+                    {
+                            putchar(0xFF & ((char*)mem)[j]);        
+                    }
+                    else /* other char */
+                    {
+                            putchar('.');
+                    }
+            }
+            putchar('\n');
+        }
+    }
+}

+ 80 - 0
poc/fastlz/ringbuffer.h

@@ -0,0 +1,80 @@
+/* Philip Thrasher's Crazy Awesome Ring Buffer Macros!
+ *
+ * Below you will find some naughty macros for easy owning and manipulating
+ * generic ring buffers. Yes, they are slightly evil in readability, but they
+ * are really fast, and they work great.
+ *
+ * Example usage:
+ *
+ * #include <stdio.h>
+ *
+ * // So we can use this in any method, this gives us a typedef
+ * // named 'intBuffer'.
+ * ringBuffer_typedef(int, intBuffer);
+ *
+ * int main() {
+ *   // Declare vars.
+ *   intBuffer myBuffer;
+ *
+ *   bufferInit(myBuffer,1024,int);
+ *
+ *   // We must have the pointer. All of the macros deal with the pointer.
+ *   // (except for init.)
+ *   intBuffer* myBuffer_ptr;
+ *   myBuffer_ptr = &myBuffer;
+ *
+ *   // Write two values.
+ *   bufferWrite(myBuffer_ptr,37);
+ *   bufferWrite(myBuffer_ptr,72);
+ *
+ *   // Read a value into a local variable.
+ *   int first;
+ *   bufferRead(myBuffer_ptr,first);
+ *   assert(first == 37); // true
+ *
+ *   int second;
+ *   bufferRead(myBuffer_ptr,second);
+ *   assert(second == 72); // true
+ *
+ *   return 0;
+ * }
+ *
+ */
+
+#ifndef _ringbuffer_h
+#define _ringbuffer_h
+
+#define ringBuffer_typedef(T, NAME) \
+  typedef struct { \
+    int size; \
+    int start; \
+    int end; \
+    T* elems; \
+  } NAME
+
+#define bufferInit(BUF, S, T) \
+  BUF.size = S+1; \
+  BUF.start = 0; \
+  BUF.end = 0; \
+  BUF.elems = (T*)calloc(BUF.size, sizeof(T))
+
+
+#define bufferDestroy(BUF) free(BUF->elems)
+#define nextStartIndex(BUF) ((BUF->start + 1) % BUF->size)
+#define nextEndIndex(BUF) ((BUF->end + 1) % BUF->size)
+#define isBufferEmpty(BUF) (BUF->end == BUF->start)
+#define isBufferFull(BUF) (nextEndIndex(BUF) == BUF->start)
+
+#define bufferWrite(BUF, ELEM) \
+  BUF->elems[BUF->end] = ELEM; \
+  BUF->end = (BUF->end + 1) % BUF->size; \
+  if (isBufferEmpty(BUF)) { \
+    BUF->start = nextStartIndex(BUF); \
+  }
+
+#define bufferRead(BUF, ELEM) \
+    ELEM = BUF->elems[BUF->start]; \
+    BUF->start = nextStartIndex(BUF);
+
+#endif
+