Initial commit

passtest.xcodeproj/project.xcworkspace/contents.xcworkspacedata

+<?xml version="1.0" encoding="UTF-8"?>
+<Workspace
+   version = "1.0">
+   <FileRef
+      location = "self:passtest.xcodeproj">
+   </FileRef>
+</Workspace>

passtest/convert.c

+//
+//  convert.c
+//  passtest
+//
+//  Created by Yury Popov on 13.08.16.
+//  Copyright © 2016 DJ PhoeniX. All rights reserved.
+//
+
+#include "passtest.h"
+#include <bzlib.h>
+#include <zlib.h>
+
+enum compression {
+  COMPRESSION_NONE,
+  COMPRESSION_BZIP2,
+  COMPRESSION_GZIP
+};
+
+struct convert_ctx {
+  enum compression comp;
+  FILE *fd_read, *fd_write;
+  size_t filesize, offset;
+  union {
+    bz_stream bz;
+    z_stream gz;
+  };
+  uint16_t index[1000000];
+  uint16_t *series[1000000];
+};
+
+int GZ_errcheck(int err) {
+  switch (err) {
+    case Z_OK:
+    case Z_STREAM_END:
+    case Z_NEED_DICT:
+      return 0;
+      
+    case Z_ERRNO:
+      fprintf(stderr, "Z_ERRNO\n");
+      return 1;
+      
+    case Z_STREAM_ERROR:
+      fprintf(stderr, "Z_STREAM_ERROR\n");
+      return 1;
+      
+    case Z_DATA_ERROR:
+      fprintf(stderr, "Z_DATA_ERROR\n");
+      return 1;
+      
+    case Z_MEM_ERROR:
+      fprintf(stderr, "Z_MEM_ERROR\n");
+      return 1;
+      
+    case Z_BUF_ERROR:
+      fprintf(stderr, "Z_BUF_ERROR\n");
+      return 1;
+      
+    case Z_VERSION_ERROR:
+      fprintf(stderr, "Z_VERSION_ERROR\n");
+      return 1;
+    default:
+      fprintf(stderr, "GZ error %d\n", err);
+      return 1;
+  }
+}
+
+int BZ_errcheck(int err) {
+  switch (err) {
+    case BZ_OK:
+    case BZ_RUN_OK:
+    case BZ_FLUSH_OK:
+    case BZ_FINISH_OK:
+    case BZ_STREAM_END:
+      return 0;
+      
+    case BZ_SEQUENCE_ERROR:
+      fprintf(stderr, "BZ_SEQUENCE_ERROR\n");
+      return 1;
+      
+    case BZ_PARAM_ERROR:
+      fprintf(stderr, "BZ_PARAM_ERROR\n");
+      return 1;
+      
+    case BZ_MEM_ERROR:
+      fprintf(stderr, "BZ_MEM_ERROR\n");
+      return 1;
+      
+    case BZ_DATA_ERROR:
+      fprintf(stderr, "BZ_DATA_ERROR\n");
+      return 1;
+      
+    case BZ_DATA_ERROR_MAGIC:
+      fprintf(stderr, "BZ_DATA_ERROR_MAGIC\n");
+      return 1;
+      
+    case BZ_IO_ERROR:
+      fprintf(stderr, "BZ_IO_ERROR\n");
+      return 1;
+      
+    case BZ_UNEXPECTED_EOF:
+      fprintf(stderr, "BZ_UNEXPECTED_EOF\n");
+      return 1;
+      
+    case BZ_OUTBUFF_FULL:
+      fprintf(stderr, "BZ_OUTBUFF_FULL\n");
+      return 1;
+      
+    case BZ_CONFIG_ERROR:
+      fprintf(stderr, "BZ_CONFIG_ERROR\n");
+      return 1;
+      
+    default:
+      fprintf(stderr, "BZ error %d\n", err);
+      return 1;
+  }
+}
+
+size_t convert_read(struct convert_ctx *ctx, void *buf, size_t size) {
+  size_t rlen;
+  if (ctx->comp == COMPRESSION_NONE) {
+    rlen = fread(buf, 1, size, ctx->fd_read);
+    ctx->offset += rlen;
+    return rlen;
+  }
+  char cbuf[size/16];
+  int res = 0;
+  switch (ctx->comp) {
+    case COMPRESSION_BZIP2:
+      ctx->bz.avail_out = (unsigned int)size;
+      ctx->bz.next_out = buf;
+      while(1) {
+        rlen = fread(cbuf, 1, sizeof(cbuf), ctx->fd_read);
+        ctx->offset += rlen;
+        ctx->bz.avail_in = (unsigned int)rlen;
+        ctx->bz.next_in = cbuf;
+        if ((BZ_errcheck(res = BZ2_bzDecompress(&ctx->bz))) != 0) return -1;
+        if (res == BZ_STREAM_END) return 0;
+        if (ctx->bz.avail_out != size)
+          return size - ctx->bz.avail_out;
+      }
+      
+    case COMPRESSION_GZIP:
+      ctx->gz.avail_out = (unsigned int)size;
+      ctx->gz.next_out = buf;
+      while(1) {
+        rlen = fread(cbuf, 1, sizeof(cbuf), ctx->fd_read);
+        ctx->offset += rlen;
+        ctx->gz.avail_in = (unsigned int)rlen;
+        ctx->gz.next_in = (Bytef*)cbuf;
+        if ((GZ_errcheck(res = inflate(&ctx->gz, Z_SYNC_FLUSH))) != 0) return -1;
+        if (res == Z_STREAM_END) return 0;
+        if (ctx->gz.avail_out != size)
+          return size - ctx->gz.avail_out;
+      }
+      
+    default:
+      return 0;
+  }
+  return -1;
+}
+
+void convert_push(struct convert_ctx *ctx, const char *serie, int sl, const char *num, int nl) {
+  if (sl >= 32 || nl >= 32) return;
+  static char s[32], n[32];
+  memcpy(s, serie, sl);
+  memcpy(n, num, nl);
+  s[sl]=0; n[nl]=0;
+  
+  char *e;
+  uint16_t si = (uint16_t)strtoul(s, &e, 10);
+  if (e != s + sl) return;
+  uint32_t ni = (uint32_t)strtoul(n, &e, 10);
+  if (e != n + nl) return;
+  
+  size_t idx = ctx->index[ni];
+  ctx->index[ni] ++;
+  if (idx % 100 == 0)
+    ctx->series[ni] = realloc(ctx->series[ni], sizeof(uint16_t) * (idx + 100));
+  ctx->series[ni][idx] = si;
+}
+
+size_t convert_parse(struct convert_ctx *ctx, const char *buf, size_t len) {
+  size_t offset = 0, ll, sl, il;
+  const char *nl, *sep, *line, *num;
+  while (1) {
+    line = buf + offset;
+    nl = strnstr(line, "\n", len - offset);
+    if (nl == NULL) break;
+    ll = nl - line;
+    sep = strnstr(line, ",", ll);
+    if (sep == NULL) break;
+    sl = sep - line;
+    num = line + sl + 1;
+    il = ll - sl - 1;
+    convert_push(ctx, line, (int)sl, num, (int)il);
+    offset += ll + 1;
+  }
+  return offset;
+}
+
+int convert_process(struct convert_ctx *ctx) {
+  char buf[1024*1024];
+  size_t len, rem = 0;
+  fseek(ctx->fd_read, 0, SEEK_END);
+  ctx->filesize = ftell(ctx->fd_read);
+  fseek(ctx->fd_read, 0, SEEK_SET);
+  while (1) {
+    len = convert_read(ctx, buf + rem, sizeof(buf) - rem) + rem;
+    if (len == rem) break;
+    if (len ==  -1) return 1;
+//    printf("%zu/%zu (%zu%%)\n", ctx->offset, ctx->filesize, ctx->offset * 100 / ctx->filesize);
+    rem = len - convert_parse(ctx, buf, len);
+    memcpy(buf, buf + len - rem, rem);
+  }
+  return 0;
+}
+
+int convert(int argc, const char * argv[]) {
+  int res = 1;
+  if (argc != 1) return usage();
+  struct convert_ctx *ctx = malloc(sizeof(struct convert_ctx));
+  if (!ctx) {
+    fprintf(stderr, "Memory error\n");
+    goto err;
+  }
+  bzero(ctx, sizeof(ctx));
+  
+  const char *csvpath = argv[0];
+  const char *ext = strrchr(csvpath, '.');
+  ctx->comp = COMPRESSION_NONE;
+  
+  ctx->fd_read = fopen(csvpath, "rb");
+  if (!ctx->fd_read) {
+    fprintf(stderr, "Cannot open file %s: %s\n", csvpath, strerror(errno));
+    goto err;
+  }
+  ctx->fd_write = fopen(dbpath, "wb");
+  if (!ctx->fd_write) {
+    fprintf(stderr, "Cannot open file %s: %s\n", csvpath, strerror(errno));
+    goto err;
+  }
+
+  if (ext != 0 && strcmp(ext, ".csv") != 0) {
+    if (strcmp(ext, ".bz2") == 0) {
+      ctx->comp = COMPRESSION_BZIP2;
+      if ((res = BZ_errcheck(BZ2_bzDecompressInit(&ctx->bz, 0, 0))) != 0) goto err;
+    }
+    else if (strcmp(ext, ".gz") == 0) {
+      ctx->comp = COMPRESSION_GZIP;
+      if ((res = GZ_errcheck(inflateInit2(&ctx->gz, 15 | 32))) != 0) goto err;
+    }
+  }
+  
+  res = convert_process(ctx);
+  if (res != 0) goto err;
+  
+  uint32_t total = 0;
+  for (uint32_t i = 0; i<1000000; i++) {
+    fwrite(&total, sizeof(total), 1, ctx->fd_write);
+    total += ctx->index[i];
+  }
+  for (uint32_t i = 0; i<1000000; i++) {
+    fwrite(ctx->series[i], sizeof(uint16_t), ctx->index[i], ctx->fd_write);
+  }
+
+err:
+  if (ctx) {
+    if (ctx->fd_read) fclose(ctx->fd_read);
+    if (ctx->fd_write) fclose(ctx->fd_write);
+    for (uint32_t i = 0; i<1000000; i++) {
+      if (ctx->series[i]) free(ctx->series[i]);
+    }
+    free(ctx);
+  }
+  return res;
+}

passtest/main.c

+//
+//  main.c
+//  passtest
+//
+//  Created by Yury Popov on 13.08.16.
+//  Copyright © 2016 DJ PhoeniX. All rights reserved.
+//
+
+#include "passtest.h"
+
+const char *dbpath = defdbpath;
+
+int main(int argc, const char * argv[]) {
+  int nargc = 0;
+  const char *nargv[argc-1];
+  const char *operation = NULL;
+  for (int i=1; i<argc; i++) {
+    if ((strcmp(argv[i],"-d") == 0) && (i < argc-1)) {
+      dbpath = argv[i+1];
+      i++;
+      continue;
+    }
+    
+    if (operation == NULL) {
+      operation = argv[i];
+      continue;
+    }
+    
+    nargv[nargc++] = argv[i];
+  }
+  
+  if (operation == NULL) return usage();
+  if (strcmp(operation, "convert") == 0) return convert(nargc, nargv);
+  if (strcmp(operation, "test") == 0) return test(nargc, nargv);
+  if (strcmp(operation, "batch") == 0) return batch(nargc, nargv);
+  return usage();
+}

passtest/passtest.h

+//
+//  passtest.h
+//  passtest
+//
+//  Created by Yury Popov on 13.08.16.
+//  Copyright © 2016 DJ PhoeniX. All rights reserved.
+//
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+static const char defdbpath[] = "db.bin";
+extern const char *dbpath;
+int usage();
+int convert(int argc, const char * argv[]);
+int test(int argc, const char * argv[]);
+int batch(int argc, const char * argv[]);

passtest/test.c

+//
+//  test.c
+//  passtest
+//
+//  Created by Yury Popov on 13.08.16.
+//  Copyright © 2016 DJ PhoeniX. All rights reserved.
+//
+
+#include "passtest.h"
+
+FILE *db_fd = NULL;
+
+int check_passport(uint16_t serie, uint32_t num) {
+  if (!db_fd) db_fd = fopen(dbpath, "rb");
+  fseek(db_fd, num * sizeof(uint32_t), SEEK_SET);
+  uint32_t off = 0, top = 0;
+  fread(&off, sizeof(off), 1, db_fd);
+  if (num < 999999) {
+    fread(&top, sizeof(top), 1, db_fd);
+  } else {
+    fseek(db_fd, 0, SEEK_END);
+    uint32_t total = (uint32_t)ftell(db_fd);
+    top = (total - 1000000 * sizeof(uint32_t)) / sizeof(uint16_t);
+  }
+  uint16_t series[top-off];
+  fseek(db_fd, 1000000 * sizeof(uint32_t) + off * sizeof(uint16_t), SEEK_SET);
+  fread(&series, top-off, sizeof(uint16_t), db_fd);
+  for (uint32_t i=0; i<top-off; i++) {
+    if (series[i] == serie) return 1;
+  }
+  return 0;
+}
+
+int check_passport_s(const char *s) {
+  static char ser[32], num[32];
+  char *se, *ne;
+  size_t ll;
+  uint16_t si; uint32_t ni;
+
+  ll = strlen(s);
+  strncpy(num, s + ll - 6, 6);
+  strncpy(ser, s, ll - 6);
+  num[6] = 0; ser[ll-6] = 0;
+  
+  si = (uint16_t)strtoul(ser, &se, 10);
+  ni = (uint32_t)strtoul(num, &ne, 10);
+  if ((se != ser + ll - 6) || (ne != num + 6)) return -1;
+  
+  return check_passport(si, ni);
+}
+
+int test(int argc, const char * argv[]) {
+  int res;
+  for (size_t i = 0; i < argc; i++) {
+    res = check_passport_s(argv[i]);
+    switch (res) {
+      case 0:
+        printf("OK: %s\n", argv[i]);
+        break;
+      case 1:
+        printf("--: %s\n", argv[i]);
+        break;
+      default:
+        printf("INVALID: %s\n", argv[i]);
+        return 1;
+    }
+  }
+  return 0;
+}
+
+int batch(int argc, const char * argv[]) {
+  if (argc > 1) return usage();
+  FILE *list_fd = NULL;
+  if (argc == 0) {
+    list_fd = fdopen(0, "r");
+  } else {
+    list_fd = fopen(argv[0], "r");
+  }
+  if (!list_fd) {
+    fprintf(stderr, "Failed to open list: %s", strerror(errno));
+    return 1;
+  }
+  size_t len = 0;
+  char *line = 0;
+  int res;
+  while ((line = fgetln(list_fd, &len)) != NULL) {
+    line[len-1] = 0;
+    res = check_passport_s(line);
+    switch (res) {
+      case 0:
+        printf("OK: %s\n", line);
+        break;
+      case 1:
+        printf("--: %s\n", line);
+        break;
+      default:
+        printf("INVALID: %s\n", line);
+        return 1;
+    }
+  }
+  return 0;
+}

passtest/usage.c

+//
+//  usage.c
+//  passtest
+//
+//  Created by Yury Popov on 13.08.16.
+//  Copyright © 2016 DJ PhoeniX. All rights reserved.
+//
+
+#include "passtest.h"
+
+int usage() {
+  fprintf(stderr,
+          "Usage:\n"
+          "          passtest [-d <DB>] download\n"
+          "          passtest [-d <DB>] convert <CSV>[.bz2 | .gz]\n"
+          "          passtest [-d <DB>] test <NUM> [<NUM> ...]\n"
+          "          passtest [-d <DB>] batch [<LIST>]\n"
+          "Arguments:\n"
+          " -d <DB>  Specifies binary database path\n"
+          "          (default: %s)\n"
+          "   <CSV>  CSV-formatted list, can be compressed\n"
+          "   <NUM>  Passport number to check\n"
+          "  <LIST>  File with passport numbers to check\n"
+          "          (default: stdin)\n"
+          ,
+          defdbpath
+          );
+  return 1;
+}