Skip to content

GitLab

P
passports
Commit fd231996 authored by PhoeniX's avatar PhoeniX
Browse files
Options

Initial commit

parents
Hide whitespace changes
Inline Side-by-side
Showing with 742 additions and 0 deletions
passtest.xcodeproj/project.pbxproj 0 → 100644
View file @ fd231996
// !$*UTF8*$!
{
archiveVersion = 1;
classes = {
};
objectVersion = 46;
objects = {
/* Begin PBXBuildFile section */
D0684B1D1D5F5A4C009E04B1 /* main.c in Sources */ = {isa = PBXBuildFile; fileRef = D0684B1C1D5F5A4C009E04B1 /* main.c */; };
D0684B241D5F5B97009E04B1 /* libc.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = D0684B231D5F5B97009E04B1 /* libc.tbd */; };
D0684B261D5F5BA3009E04B1 /* libbz2.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = D0684B251D5F5BA3009E04B1 /* libbz2.tbd */; };
D0684B281D5F5F2A009E04B1 /* convert.c in Sources */ = {isa = PBXBuildFile; fileRef = D0684B271D5F5F2A009E04B1 /* convert.c */; };
D0684B2D1D5F5F6D009E04B1 /* test.c in Sources */ = {isa = PBXBuildFile; fileRef = D0684B2B1D5F5F6D009E04B1 /* test.c */; };
D0684B301D5F601A009E04B1 /* usage.c in Sources */ = {isa = PBXBuildFile; fileRef = D0684B2F1D5F601A009E04B1 /* usage.c */; };
D0684B321D5F64EC009E04B1 /* libz.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = D0684B311D5F64EC009E04B1 /* libz.tbd */; };
/* End PBXBuildFile section */
/* Begin PBXCopyFilesBuildPhase section */
D0684B171D5F5A4C009E04B1 /* CopyFiles */ = {
isa = PBXCopyFilesBuildPhase;
buildActionMask = 2147483647;
dstPath = /usr/share/man/man1/;
dstSubfolderSpec = 0;
files = (
);
runOnlyForDeploymentPostprocessing = 1;
};
/* End PBXCopyFilesBuildPhase section */
/* Begin PBXFileReference section */
D0684B191D5F5A4C009E04B1 /* passtest */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = passtest; sourceTree = BUILT_PRODUCTS_DIR; };
D0684B1C1D5F5A4C009E04B1 /* main.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = main.c; sourceTree = "<group>"; };
D0684B231D5F5B97009E04B1 /* libc.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libc.tbd; path = usr/lib/libc.tbd; sourceTree = SDKROOT; };
D0684B251D5F5BA3009E04B1 /* libbz2.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libbz2.tbd; path = usr/lib/libbz2.tbd; sourceTree = SDKROOT; };
D0684B271D5F5F2A009E04B1 /* convert.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = convert.c; sourceTree = "<group>"; };
D0684B2B1D5F5F6D009E04B1 /* test.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = test.c; sourceTree = "<group>"; };
D0684B2E1D5F5FB0009E04B1 /* passtest.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = passtest.h; sourceTree = "<group>"; };
D0684B2F1D5F601A009E04B1 /* usage.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = usage.c; sourceTree = "<group>"; };
D0684B311D5F64EC009E04B1 /* libz.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libz.tbd; path = usr/lib/libz.tbd; sourceTree = SDKROOT; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
D0684B161D5F5A4C009E04B1 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
D0684B321D5F64EC009E04B1 /* libz.tbd in Frameworks */,
D0684B261D5F5BA3009E04B1 /* libbz2.tbd in Frameworks */,
D0684B241D5F5B97009E04B1 /* libc.tbd in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXFrameworksBuildPhase section */
/* Begin PBXGroup section */
D0684B101D5F5A4C009E04B1 = {
isa = PBXGroup;
children = (
D0684B311D5F64EC009E04B1 /* libz.tbd */,
D0684B251D5F5BA3009E04B1 /* libbz2.tbd */,
D0684B231D5F5B97009E04B1 /* libc.tbd */,
D0684B1B1D5F5A4C009E04B1 /* passtest */,
D0684B1A1D5F5A4C009E04B1 /* Products */,
);
sourceTree = "<group>";
};
D0684B1A1D5F5A4C009E04B1 /* Products */ = {
isa = PBXGroup;
children = (
D0684B191D5F5A4C009E04B1 /* passtest */,
);
name = Products;
sourceTree = "<group>";
};
D0684B1B1D5F5A4C009E04B1 /* passtest */ = {
isa = PBXGroup;
children = (
D0684B2E1D5F5FB0009E04B1 /* passtest.h */,
D0684B1C1D5F5A4C009E04B1 /* main.c */,
D0684B271D5F5F2A009E04B1 /* convert.c */,
D0684B2B1D5F5F6D009E04B1 /* test.c */,
D0684B2F1D5F601A009E04B1 /* usage.c */,
);
path = passtest;
sourceTree = "<group>";
};
/* End PBXGroup section */
/* Begin PBXNativeTarget section */
D0684B181D5F5A4C009E04B1 /* passtest */ = {
isa = PBXNativeTarget;
buildConfigurationList = D0684B201D5F5A4C009E04B1 /* Build configuration list for PBXNativeTarget "passtest" */;
buildPhases = (
D0684B151D5F5A4C009E04B1 /* Sources */,
D0684B161D5F5A4C009E04B1 /* Frameworks */,
D0684B171D5F5A4C009E04B1 /* CopyFiles */,
);
buildRules = (
);
dependencies = (
);
name = passtest;
productName = passtest;
productReference = D0684B191D5F5A4C009E04B1 /* passtest */;
productType = "com.apple.product-type.tool";
};
/* End PBXNativeTarget section */
/* Begin PBXProject section */
D0684B111D5F5A4C009E04B1 /* Project object */ = {
isa = PBXProject;
attributes = {
LastUpgradeCheck = 0730;
ORGANIZATIONNAME = "DJ PhoeniX";
TargetAttributes = {
D0684B181D5F5A4C009E04B1 = {
CreatedOnToolsVersion = 7.3.1;
};
};
};
buildConfigurationList = D0684B141D5F5A4C009E04B1 /* Build configuration list for PBXProject "passtest" */;
compatibilityVersion = "Xcode 3.2";
developmentRegion = English;
hasScannedForEncodings = 0;
knownRegions = (
en,
);
mainGroup = D0684B101D5F5A4C009E04B1;
productRefGroup = D0684B1A1D5F5A4C009E04B1 /* Products */;
projectDirPath = "";
projectRoot = "";
targets = (
D0684B181D5F5A4C009E04B1 /* passtest */,
);
};
/* End PBXProject section */
/* Begin PBXSourcesBuildPhase section */
D0684B151D5F5A4C009E04B1 /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
D0684B301D5F601A009E04B1 /* usage.c in Sources */,
D0684B1D1D5F5A4C009E04B1 /* main.c in Sources */,
D0684B281D5F5F2A009E04B1 /* convert.c in Sources */,
D0684B2D1D5F5F6D009E04B1 /* test.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXSourcesBuildPhase section */
/* Begin XCBuildConfiguration section */
D0684B1E1D5F5A4C009E04B1 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_ANALYZER_NONNULL = YES;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
CLANG_CXX_LIBRARY = "libc++";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
CODE_SIGN_IDENTITY = "-";
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = dwarf;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_TESTABILITY = YES;
GCC_C_LANGUAGE_STANDARD = gnu99;
GCC_DYNAMIC_NO_PIC = NO;
GCC_NO_COMMON_BLOCKS = YES;
GCC_OPTIMIZATION_LEVEL = 0;
GCC_PREPROCESSOR_DEFINITIONS = (
"DEBUG=1",
"$(inherited)",
);
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
MACOSX_DEPLOYMENT_TARGET = 10.11;
MTL_ENABLE_DEBUG_INFO = YES;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = macosx;
};
name = Debug;
};
D0684B1F1D5F5A4C009E04B1 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_ANALYZER_NONNULL = YES;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
CLANG_CXX_LIBRARY = "libc++";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
CODE_SIGN_IDENTITY = "-";
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
ENABLE_NS_ASSERTIONS = NO;
ENABLE_STRICT_OBJC_MSGSEND = YES;
GCC_C_LANGUAGE_STANDARD = gnu99;
GCC_NO_COMMON_BLOCKS = YES;
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
MACOSX_DEPLOYMENT_TARGET = 10.11;
MTL_ENABLE_DEBUG_INFO = NO;
SDKROOT = macosx;
};
name = Release;
};
D0684B211D5F5A4C009E04B1 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
PRODUCT_NAME = "$(TARGET_NAME)";
};
name = Debug;
};
D0684B221D5F5A4C009E04B1 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
PRODUCT_NAME = "$(TARGET_NAME)";
};
name = Release;
};
/* End XCBuildConfiguration section */
/* Begin XCConfigurationList section */
D0684B141D5F5A4C009E04B1 /* Build configuration list for PBXProject "passtest" */ = {
isa = XCConfigurationList;
buildConfigurations = (
D0684B1E1D5F5A4C009E04B1 /* Debug */,
D0684B1F1D5F5A4C009E04B1 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
D0684B201D5F5A4C009E04B1 /* Build configuration list for PBXNativeTarget "passtest" */ = {
isa = XCConfigurationList;
buildConfigurations = (
D0684B211D5F5A4C009E04B1 /* Debug */,
D0684B221D5F5A4C009E04B1 /* Release */,
);
defaultConfigurationIsVisible = 0;
};
/* End XCConfigurationList section */
};
rootObject = D0684B111D5F5A4C009E04B1 /* Project object */;
}
passtest.xcodeproj/project.xcworkspace/contents.xcworkspacedata 0 → 100644
View file @ fd231996
<?xml version="1.0" encoding="UTF-8"?>
<Workspace
version = "1.0">
<FileRef
location = "self:passtest.xcodeproj">
</FileRef>
</Workspace>
passtest/convert.c 0 → 100644
View file @ fd231996
//
// convert.c
// passtest
//
// Created by Yury Popov on 13.08.16.
// Copyright © 2016 DJ PhoeniX. All rights reserved.
//
#include "passtest.h"
#include <bzlib.h>
#include <zlib.h>
enum compression {
COMPRESSION_NONE,
COMPRESSION_BZIP2,
COMPRESSION_GZIP
};
struct convert_ctx {
enum compression comp;
FILE *fd_read, *fd_write;
size_t filesize, offset;
union {
bz_stream bz;
z_stream gz;
};
uint16_t index[1000000];
uint16_t *series[1000000];
};
int GZ_errcheck(int err) {
switch (err) {
case Z_OK:
case Z_STREAM_END:
case Z_NEED_DICT:
return 0;
case Z_ERRNO:
fprintf(stderr, "Z_ERRNO\n");
return 1;
case Z_STREAM_ERROR:
fprintf(stderr, "Z_STREAM_ERROR\n");
return 1;
case Z_DATA_ERROR:
fprintf(stderr, "Z_DATA_ERROR\n");
return 1;
case Z_MEM_ERROR:
fprintf(stderr, "Z_MEM_ERROR\n");
return 1;
case Z_BUF_ERROR:
fprintf(stderr, "Z_BUF_ERROR\n");
return 1;
case Z_VERSION_ERROR:
fprintf(stderr, "Z_VERSION_ERROR\n");
return 1;
default:
fprintf(stderr, "GZ error %d\n", err);
return 1;
}
}
int BZ_errcheck(int err) {
switch (err) {
case BZ_OK:
case BZ_RUN_OK:
case BZ_FLUSH_OK:
case BZ_FINISH_OK:
case BZ_STREAM_END:
return 0;
case BZ_SEQUENCE_ERROR:
fprintf(stderr, "BZ_SEQUENCE_ERROR\n");
return 1;
case BZ_PARAM_ERROR:
fprintf(stderr, "BZ_PARAM_ERROR\n");
return 1;
case BZ_MEM_ERROR:
fprintf(stderr, "BZ_MEM_ERROR\n");
return 1;
case BZ_DATA_ERROR:
fprintf(stderr, "BZ_DATA_ERROR\n");
return 1;
case BZ_DATA_ERROR_MAGIC:
fprintf(stderr, "BZ_DATA_ERROR_MAGIC\n");
return 1;
case BZ_IO_ERROR:
fprintf(stderr, "BZ_IO_ERROR\n");
return 1;
case BZ_UNEXPECTED_EOF:
fprintf(stderr, "BZ_UNEXPECTED_EOF\n");
return 1;
case BZ_OUTBUFF_FULL:
fprintf(stderr, "BZ_OUTBUFF_FULL\n");
return 1;
case BZ_CONFIG_ERROR:
fprintf(stderr, "BZ_CONFIG_ERROR\n");
return 1;
default:
fprintf(stderr, "BZ error %d\n", err);
return 1;
}
}
size_t convert_read(struct convert_ctx *ctx, void *buf, size_t size) {
size_t rlen;
if (ctx->comp == COMPRESSION_NONE) {
rlen = fread(buf, 1, size, ctx->fd_read);
ctx->offset += rlen;
return rlen;
}
char cbuf[size/16];
int res = 0;
switch (ctx->comp) {
case COMPRESSION_BZIP2:
ctx->bz.avail_out = (unsigned int)size;
ctx->bz.next_out = buf;
while(1) {
rlen = fread(cbuf, 1, sizeof(cbuf), ctx->fd_read);
ctx->offset += rlen;
ctx->bz.avail_in = (unsigned int)rlen;
ctx->bz.next_in = cbuf;
if ((BZ_errcheck(res = BZ2_bzDecompress(&ctx->bz))) != 0) return -1;
if (res == BZ_STREAM_END) return 0;
if (ctx->bz.avail_out != size)
return size - ctx->bz.avail_out;
}
case COMPRESSION_GZIP:
ctx->gz.avail_out = (unsigned int)size;
ctx->gz.next_out = buf;
while(1) {
rlen = fread(cbuf, 1, sizeof(cbuf), ctx->fd_read);
ctx->offset += rlen;
ctx->gz.avail_in = (unsigned int)rlen;
ctx->gz.next_in = (Bytef*)cbuf;
if ((GZ_errcheck(res = inflate(&ctx->gz, Z_SYNC_FLUSH))) != 0) return -1;
if (res == Z_STREAM_END) return 0;
if (ctx->gz.avail_out != size)
return size - ctx->gz.avail_out;
}
default:
return 0;
}
return -1;
}
void convert_push(struct convert_ctx *ctx, const char *serie, int sl, const char *num, int nl) {
if (sl >= 32 || nl >= 32) return;
static char s[32], n[32];
memcpy(s, serie, sl);
memcpy(n, num, nl);
s[sl]=0; n[nl]=0;
char *e;
uint16_t si = (uint16_t)strtoul(s, &e, 10);
if (e != s + sl) return;
uint32_t ni = (uint32_t)strtoul(n, &e, 10);
if (e != n + nl) return;
size_t idx = ctx->index[ni];
ctx->index[ni] ++;
if (idx % 100 == 0)
ctx->series[ni] = realloc(ctx->series[ni], sizeof(uint16_t) * (idx + 100));
ctx->series[ni][idx] = si;
}
size_t convert_parse(struct convert_ctx *ctx, const char *buf, size_t len) {
size_t offset = 0, ll, sl, il;
const char *nl, *sep, *line, *num;
while (1) {
line = buf + offset;
nl = strnstr(line, "\n", len - offset);
if (nl == NULL) break;
ll = nl - line;
sep = strnstr(line, ",", ll);
if (sep == NULL) break;
sl = sep - line;
num = line + sl + 1;
il = ll - sl - 1;
convert_push(ctx, line, (int)sl, num, (int)il);
offset += ll + 1;
}
return offset;
}
int convert_process(struct convert_ctx *ctx) {
char buf[1024*1024];
size_t len, rem = 0;
fseek(ctx->fd_read, 0, SEEK_END);
ctx->filesize = ftell(ctx->fd_read);
fseek(ctx->fd_read, 0, SEEK_SET);
while (1) {
len = convert_read(ctx, buf + rem, sizeof(buf) - rem) + rem;
if (len == rem) break;
if (len == -1) return 1;
// printf("%zu/%zu (%zu%%)\n", ctx->offset, ctx->filesize, ctx->offset * 100 / ctx->filesize);
rem = len - convert_parse(ctx, buf, len);
memcpy(buf, buf + len - rem, rem);
}
return 0;
}
int convert(int argc, const char * argv[]) {
int res = 1;
if (argc != 1) return usage();
struct convert_ctx *ctx = malloc(sizeof(struct convert_ctx));
if (!ctx) {
fprintf(stderr, "Memory error\n");
goto err;
}
bzero(ctx, sizeof(ctx));
const char *csvpath = argv[0];
const char *ext = strrchr(csvpath, '.');
ctx->comp = COMPRESSION_NONE;
ctx->fd_read = fopen(csvpath, "rb");
if (!ctx->fd_read) {
fprintf(stderr, "Cannot open file %s: %s\n", csvpath, strerror(errno));
goto err;
}
ctx->fd_write = fopen(dbpath, "wb");
if (!ctx->fd_write) {
fprintf(stderr, "Cannot open file %s: %s\n", csvpath, strerror(errno));
goto err;
}
if (ext != 0 && strcmp(ext, ".csv") != 0) {
if (strcmp(ext, ".bz2") == 0) {
ctx->comp = COMPRESSION_BZIP2;
if ((res = BZ_errcheck(BZ2_bzDecompressInit(&ctx->bz, 0, 0))) != 0) goto err;
}
else if (strcmp(ext, ".gz") == 0) {
ctx->comp = COMPRESSION_GZIP;
if ((res = GZ_errcheck(inflateInit2(&ctx->gz, 15 | 32))) != 0) goto err;
}
}
res = convert_process(ctx);
if (res != 0) goto err;
uint32_t total = 0;
for (uint32_t i = 0; i<1000000; i++) {
fwrite(&total, sizeof(total), 1, ctx->fd_write);
total += ctx->index[i];
}
for (uint32_t i = 0; i<1000000; i++) {
fwrite(ctx->series[i], sizeof(uint16_t), ctx->index[i], ctx->fd_write);
}
err:
if (ctx) {
if (ctx->fd_read) fclose(ctx->fd_read);
if (ctx->fd_write) fclose(ctx->fd_write);
for (uint32_t i = 0; i<1000000; i++) {
if (ctx->series[i]) free(ctx->series[i]);
}
free(ctx);
}
return res;
}
passtest/main.c 0 → 100644
View file @ fd231996
//
// main.c
// passtest
//
// Created by Yury Popov on 13.08.16.
// Copyright © 2016 DJ PhoeniX. All rights reserved.
//
#include "passtest.h"
const char *dbpath = defdbpath;
int main(int argc, const char * argv[]) {
int nargc = 0;
const char *nargv[argc-1];
const char *operation = NULL;
for (int i=1; i<argc; i++) {
if ((strcmp(argv[i],"-d") == 0) && (i < argc-1)) {
dbpath = argv[i+1];
i++;
continue;
}
if (operation == NULL) {
operation = argv[i];
continue;
}
nargv[nargc++] = argv[i];
}
if (operation == NULL) return usage();
if (strcmp(operation, "convert") == 0) return convert(nargc, nargv);
if (strcmp(operation, "test") == 0) return test(nargc, nargv);
if (strcmp(operation, "batch") == 0) return batch(nargc, nargv);
return usage();
}
passtest/passtest.h 0 → 100644
View file @ fd231996
//
// passtest.h