Merge pull request #11 from heios/master

pseudo-yaml output/processing functionality

Makefile.in

@@ -22,14 +22,22 @@ ED25519OBJ= $(ED25519_@ED25519IMPL@)
 
 MAINOBJ= \
 	main.c.o \
+	yaml.c.o \
 	vec.c.o \
 	cpucount.c.o \
 	base32_to.c.o \
 	base32_from.c.o \
+	base64_to.c.o \
+	base64_from.c.o \
 	ioutil.c.o \
 	$(ED25519OBJ) \
 	keccak.c.o
 
+TEST_BASE64OBJ= \
+	test_base64.c.o \
+	base64_to.c.o \
+	base64_from.c.o
+
 TEST_BASE32OBJ= \
 	test_base32.c.o \
 	base32_to.c.o \
@@ -48,6 +56,7 @@ TEST_ED25519OBJ= \
 
 ALLO= $(sort \
 	$(MAINOBJ) \
+	$(TEST_BASE64OBJ) \
 	$(TEST_BASE32OBJ) \
 	$(TEST_BASE16OBJ) \
 	$(TEST_ED25519OBJ) \
@@ -60,7 +69,7 @@ CLEANO= $(filter %.o,$(ALLO))
 MAINLIB= -lpthread -lsodium @MAINLIB@
 TEST_ED25519LIB= -lsodium
 
-EXE= mkp224o test_base32 test_base16 test_ed25519
+EXE= mkp224o test_base64 test_base32 test_base16 test_ed25519
 
 default: mkp224o
 
@@ -69,6 +78,9 @@ all: $(EXE)
 mkp224o: $(MAINOBJ)
 	$(CC) $(LDFLAGS) $(CFLAGS) -o $@.tmp $^ $(MAINLIB) && $(MV) $@.tmp $@
 
+test_base64: $(TEST_BASE64OBJ)
+	$(CC) $(LDFLAGS) $(CFLAGS) -o $@.tmp $^ && $(MV) $@.tmp $@
+
 test_base32: $(TEST_BASE32OBJ)
 	$(CC) $(LDFLAGS) $(CFLAGS) -o $@.tmp $^ && $(MV) $@.tmp $@
 
@@ -104,6 +116,8 @@ base16_from.c.o: types.h base16.h
 base16_to.c.o: types.h base16.h
 base32_from.c.o: types.h base32.h
 base32_to.c.o: types.h base32.h
+base64_from.c.o: types.h base64.h
+base64_to.c.o: types.h base64.h
 cpucount.c.o: cpucount.h
 ed25519/amd64-51-30k/batch.c.o: ed25519/amd64-51-30k/crypto_sign.h
 ed25519/amd64-51-30k/batch.c.o: ed25519/amd64-51-30k/ed25519.h
@@ -381,9 +395,11 @@ main.c.o: ed25519/ed25519-donna/modm-donna-64bit.h
 main.c.o: ed25519/ed25519-donna/ed25519-donna-basepoint-table.h
 main.c.o: ed25519/ed25519-donna/ed25519-donna-64bit-tables.h
 main.c.o: ed25519/ed25519-donna/ed25519-donna-64bit-x86.h
-main.c.o: ed25519/ed25519-donna/ed25519-donna-impl-base.h ioutil.h filters.h
+main.c.o: ed25519/ed25519-donna/ed25519-donna-impl-base.h ioutil.h common.h
+main.c.o: yaml.h filters.h
 test_base16.c.o: types.h base16.h
 test_base32.c.o: types.h base32.h
+test_base64.c.o: types.h base64.h
 test_ed25519.c.o: types.h base16.h ed25519/ed25519.h ed25519/ref10/ed25519.h
 test_ed25519.c.o: ed25519/ref10/ge.h ed25519/ref10/fe.h
 test_ed25519.c.o: ed25519/ref10/crypto_int32.h ed25519/amd64-51-30k/ed25519.h
@@ -403,3 +419,4 @@ test_ed25519.c.o: ed25519/ed25519-donna/ed25519-donna-64bit-tables.h
 test_ed25519.c.o: ed25519/ed25519-donna/ed25519-donna-64bit-x86.h
 test_ed25519.c.o: ed25519/ed25519-donna/ed25519-donna-impl-base.h
 vec.c.o: vec.h
+yaml.c.o: types.h yaml.h ioutil.h base64.h common.h

base64.h

@@ -0,0 +1,11 @@
+// converts src[0:slen] to base64 string
+char *base64_to(char *dst,const u8 *src,size_t slen);
+// calculates length needed to store data converted to base64
+#define BASE64_TO_LEN(l) (((l + 3 - 1) / 3) * 4)
+// converts src string from base64
+size_t base64_from(u8 *dst,const char *src,size_t slen);
+// calculates length needed to store data converted from base
+#define BASE64_FROM_LEN(l) ((l) / 4 * 3)
+// validates base32 string and optionally stores length of valid data
+// returns 1 if whole string is good, 0 if string contains invalid data
+int base64_valid(const char *src,size_t *count);

base64_from.c

@@ -0,0 +1,93 @@
+#include <stddef.h>
+#include <stdint.h>
+#include "types.h"
+#include "base64.h"
+
+static const u8 base64f[256] = {
+	//00    01    02    03    04    05    06    07
+	//08    09    0A    0B    0C    0D    0E    0F
+	// 0x00..0x3F
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x00
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x08
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x10
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x18
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x20
+	0xFF, 0xFF, 0xFF, 0x3E, 0xFF, 0xFF, 0xFF, 0x3F, // 0x28
+	0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, // 0x30
+	0x3C, 0x3D, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x38
+	// 0x40..0x7F
+	0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, // 0x40
+	0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, // 0x48
+	0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, // 0x50
+	0x17, 0x18, 0x19, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x58
+	0xFF, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, // 0x60
+	0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, // 0x68
+	0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, // 0x70
+	0x31, 0x32, 0x33, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x78
+	// 0x80..0xBF
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	// 0xC0..0xFF
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+};
+
+size_t base64_from(u8 *dst,const char *src,size_t srclen)
+{
+	if (srclen % 4) {
+		return -1;
+	} else if (!srclen) {
+		return 0;
+	}
+
+	size_t dstlen = BASE64_FROM_LEN(srclen);
+	dstlen -= (src[srclen - 1] == '=');
+	dstlen -= (src[srclen - 2] == '=');
+
+	for (size_t i = 0, j = 0; i < srclen;) {
+		u32 sixbits[4];
+
+		sixbits[0] = base64f[(unsigned char)src[i++]];
+		sixbits[1] = base64f[(unsigned char)src[i++]];
+		sixbits[2] = (src[i] == '=' ? (0 & i++) : base64f[(unsigned char)src[i++]]);
+		sixbits[3] = (src[i] == '=' ? (0 & i++) : base64f[(unsigned char)src[i++]]);
+
+		u32 threebytes = 0
+			| (sixbits[0] << (3 * 6))
+			| (sixbits[1] << (2 * 6))
+			| (sixbits[2] << (1 * 6))
+			| (sixbits[3] << (0 * 6));
+
+		if (j < dstlen) dst[j++] = (threebytes >> (2 * 8));
+		if (j < dstlen) dst[j++] = (threebytes >> (1 * 8)) & 0xff;
+		if (j < dstlen) dst[j++] = (threebytes >> (0 * 8)) & 0xff;
+	}
+	return dstlen;
+}
+
+int base64_valid(const char *src,size_t *count)
+{
+	const char *p;
+
+	for (p = src;base64f[(u8)*p] != 0xFF;++p)
+		;
+
+	for (;*p == '=';++p)
+		;
+
+	if (count)
+		*count = (size_t) (p - src);
+	return !*p;
+}

base64_to.c

@@ -0,0 +1,51 @@
+#include <stddef.h>
+#include <stdint.h>
+#include "types.h"
+#include "base64.h"
+
+static const char base64t[64] = {
+	'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
+	'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
+	'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
+	'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
+	'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
+	'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
+	'w', 'x', 'y', 'z', '0', '1', '2', '3',
+	'4', '5', '6', '7', '8', '9', '+', '/',
+};
+
+char *base64_to(char *dst,const u8 *src,size_t slen)
+{
+	if (!slen) {
+		*dst = '\0';
+		return dst;
+	}
+
+	for(size_t i = 0; i < slen;) {
+		u32 threebytes = 0;
+		threebytes |= (i < slen ? (unsigned char)src[i++] : (unsigned char)0) << (2 * 8);
+		threebytes |= (i < slen ? (unsigned char)src[i++] : (unsigned char)0) << (1 * 8);
+		threebytes |= (i < slen ? (unsigned char)src[i++] : (unsigned char)0) << (0 * 8);
+
+		*dst++ = base64t[(threebytes >> (3 * 6)) & 63];
+		*dst++ = base64t[(threebytes >> (2 * 6)) & 63];
+		*dst++ = base64t[(threebytes >> (1 * 6)) & 63];
+		*dst++ = base64t[(threebytes >> (0 * 6)) & 63];
+	}
+
+	switch (slen % 3) {
+		case 0 : break;
+		case 1 : {
+			*(dst-2) = '=';
+			*(dst-1) = '=';
+			break;
+		}
+		case 2 : {
+			*(dst-1) = '=';
+			break;
+		}
+	}
+
+	*dst = '\0';
+	return dst;
+}

common.h

@@ -0,0 +1,22 @@
+#define SECRET_LEN 64
+#define PUBLIC_LEN 32
+#define SEED_LEN   32
+// with checksum + version num
+#define PUBONION_LEN (PUBLIC_LEN + 3)
+
+#define PKPREFIX_SIZE (29 + 3)
+#define SKPREFIX_SIZE (29 + 3)
+
+#define FORMATTED_PUBLIC_LEN (PKPREFIX_SIZE + PUBLIC_LEN)
+#define FORMATTED_SECRET_LEN (SKPREFIX_SIZE + SECRET_LEN)
+
+// full onion address, WITHOUT newline
+#define ONION_LEN 62
+
+extern pthread_mutex_t fout_mutex;
+extern FILE *fout;
+
+extern size_t onionendpos;   // end of .onion within string
+extern size_t direndpos;     // end of dir before .onion within string
+extern size_t printstartpos; // where to start printing from
+extern size_t printlen;      // precalculated, related to printstartpos

main.c

@@ -6,7 +6,6 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <string.h>
-#include <time.h>
 #include <pthread.h>
 #include <signal.h>
 #include <sodium/randombytes.h>
@@ -20,6 +19,8 @@
 #include "keccak.h"
 #include "ed25519/ed25519.h"
 #include "ioutil.h"
+#include "common.h"
+#include "yaml.h"
 
 #ifndef _WIN32
 #define FSZ "%zu"
@@ -29,11 +30,10 @@
 
 // additional 0 terminator is added by C
 static const char * const pkprefix = "== ed25519v1-public: type0 ==\0\0";
-#define pkprefixlen (29 + 3)
 static const char * const skprefix = "== ed25519v1-secret: type0 ==\0\0";
-#define skprefixlen (29 + 3)
+
-static const char * const checksumstr = ".onion checksum";
+static const char checksumstr[] = ".onion checksum";
-#define checksumstrlen 15
+#define checksumstrlen (sizeof(checksumstr) - 1) // 15
 
 // output directory
 static char *workdir = 0;
@@ -43,27 +43,25 @@ static int quietflag = 0;
 //static int wantdedup = 0;
 #define wantdedup 0
 
-#define SECRET_LEN 64
+// 0, direndpos, onionendpos
-#define PUBLIC_LEN 32
+// printstartpos = either 0 or direndpos
-#define SEED_LEN   32
+// printlen      = either onionendpos + 1 or ONION_LEN + 1 (additional 1 is for newline)
-// with checksum + version num
+size_t onionendpos;   // end of .onion within string
-#define PUBONION_LEN (PUBLIC_LEN + 3)
+size_t direndpos;     // end of dir before .onion within string
-// with newline included
+size_t printstartpos; // where to start printing from
-#define ONIONLEN 62
+size_t printlen;      // precalculated, related to printstartpos
 
-static size_t onionendpos;   // end of .onion within string
+static int yamloutput = 0;
-static size_t direndpos;     // end of dir before .onion within string
-static size_t printstartpos; // where to start printing from
-static size_t printlen;      // precalculated, related to printstartpos
-
-static pthread_mutex_t fout_mutex;
-static FILE *fout;
-static size_t numneedgenerate = 0;
 static int numwords = 1;
+static size_t numneedgenerate = 0;
+
 static pthread_mutex_t keysgenerated_mutex;
 static volatile size_t keysgenerated = 0;
 static volatile int endwork = 0;
 
+pthread_mutex_t fout_mutex;
+FILE *fout;
+
 static void termhandler(int sig)
 {
 	switch (sig) {
@@ -111,7 +109,6 @@ struct tstatstruct {
 VEC_STRUCT(tstatsvec,struct tstatstruct);
 #endif
 
-
 static void onionready(char *sname,const u8 *secret,const u8 *pubonion)
 {
 	if (endwork)
@@ -123,42 +120,60 @@ static void onionready(char *sname,const u8 *secret,const u8 *pubonion)
 			pthread_mutex_unlock(&keysgenerated_mutex);
 			return;
 		}
-	}
-
-	if (createdir(sname,1) != 0) {
-		if (numneedgenerate)
-			pthread_mutex_unlock(&keysgenerated_mutex);
-		return;
-	}
-
-	if (numneedgenerate) {
 		++keysgenerated;
-		if (keysgenerated >= numneedgenerate)
+		if (keysgenerated == numneedgenerate)
 			endwork = 1;
 		pthread_mutex_unlock(&keysgenerated_mutex);
 	}
 
-	strcpy(&sname[onionendpos],"/hs_ed25519_secret_key");
+	if (!yamloutput) {
-	writetofile(sname,secret,skprefixlen + SECRET_LEN,1);
+		if (createdir(sname,1) != 0) {
+			pthread_mutex_lock(&fout_mutex);
+			fprintf(stderr,"ERROR: could not create directory for key output\n");
+			pthread_mutex_unlock(&fout_mutex);
+			return;
+		}
 
-	strcpy(&sname[onionendpos],"/hostname");
+		strcpy(&sname[onionendpos],"/hs_ed25519_secret_key");
-	FILE *hfile = fopen(sname,"w");
+		writetofile(sname,secret,FORMATTED_SECRET_LEN,1);
-	if (hfile) {
+
+		strcpy(&sname[onionendpos],"/hs_ed25519_public_key");
+		writetofile(sname,pubonion,FORMATTED_PUBLIC_LEN,0);
+
+		strcpy(&sname[onionendpos],"/hostname");
+		FILE *hfile = fopen(sname,"w");
 		sname[onionendpos] = '\n';
-		fwrite(&sname[direndpos],ONIONLEN + 1,1,hfile);
+		if (hfile) {
-		fclose(hfile);
+			fwrite(&sname[direndpos],ONION_LEN + 1,1,hfile);
-	}
+			fclose(hfile);
+		}
+		if (fout) {
+			pthread_mutex_lock(&fout_mutex);
+			fwrite(&sname[printstartpos],printlen,1,fout);
+			fflush(fout);
+			pthread_mutex_unlock(&fout_mutex);
+		}
+	} else
+		yamlout_writekeys(&sname[direndpos],pubonion,secret);
+}
 
-	strcpy(&sname[onionendpos],"/hs_ed25519_public_key");
+union pubonionunion {
-	writetofile(sname,pubonion,pkprefixlen + PUBLIC_LEN,0);
+	u8 raw[PKPREFIX_SIZE + PUBLIC_LEN + 32];
+	struct {
+		u64 prefix[4];
+		u64 key[4];
+		u64 hash[4];
+	} i;
+} ;
 
-	if (fout) {
+static char *makesname()
-		sname[onionendpos] = '\n';
+{
-		pthread_mutex_lock(&fout_mutex);
+	char *sname = (char *) malloc(workdirlen + ONION_LEN + 63 + 1);
-		fwrite(&sname[printstartpos],printlen,1,fout);
+	if (!sname)
-		fflush(fout);
+		abort();
-		pthread_mutex_unlock(&fout_mutex);
+	if (workdir)
-	}
+		memcpy(sname,workdir,workdirlen);
+	return sname;
 }
 
 // little endian inc
@@ -188,17 +203,10 @@ static inline void shiftpk(u8 *dst,const u8 *src,size_t sbits)
 
 static void *dowork(void *task)
 {
-	union pubonionunion {
+	union pubonionunion pubonion;
-		u8 raw[pkprefixlen + PUBLIC_LEN + 32];
+	u8 * const pk = &pubonion.raw[PKPREFIX_SIZE];
-		struct {
+	u8 secret[SKPREFIX_SIZE + SECRET_LEN];
-			u64 prefix[4];
+	u8 * const sk = &secret[SKPREFIX_SIZE];
-			u64 key[4];
-			u64 hash[4];
-		} i;
-	} pubonion;
-	u8 * const pk = &pubonion.raw[pkprefixlen];
-	u8 secret[skprefixlen + SECRET_LEN];
-	u8 * const sk = &secret[skprefixlen];
 	u8 seed[SEED_LEN];
 	u8 hashsrc[checksumstrlen + PUBLIC_LEN + 1];
 	u8 wpk[PUBLIC_LEN + 1];
@@ -209,19 +217,15 @@ static void *dowork(void *task)
 #endif
 	PREFILTER
 
-	memcpy(secret,skprefix,skprefixlen);
+	memcpy(secret,skprefix,SKPREFIX_SIZE);
 	wpk[PUBLIC_LEN] = 0;
 	memset(&pubonion,0,sizeof(pubonion));
-	memcpy(pubonion.raw,pkprefix,pkprefixlen);
+	memcpy(pubonion.raw,pkprefix,PKPREFIX_SIZE);
 	// write version later as it will be overwritten by hash
 	memcpy(hashsrc,checksumstr,checksumstrlen);
 	hashsrc[checksumstrlen + PUBLIC_LEN] = 0x03; // version
 
-	sname = (char *) malloc(workdirlen + ONIONLEN + 63 + 1);
+	sname = makesname();
-	if (!sname)
-		abort();
-	if (workdir)
-		memcpy(sname,workdir,workdirlen);
 
 initseed:
 	randombytes(seed,sizeof(seed));
@@ -295,17 +299,10 @@ static void addsztoscalar32(u8 *dst,size_t v)
 
 static void *dofastwork(void *task)
 {
-	union pubonionunion {
+	union pubonionunion pubonion;
-		u8 raw[pkprefixlen + PUBLIC_LEN + 32];
+	u8 * const pk = &pubonion.raw[PKPREFIX_SIZE];
-		struct {
+	u8 secret[SKPREFIX_SIZE + SECRET_LEN];
-			u64 prefix[4];
+	u8 * const sk = &secret[SKPREFIX_SIZE];
-			u64 key[4];
-			u64 hash[4];
-		} i;
-	} pubonion;
-	u8 * const pk = &pubonion.raw[pkprefixlen];
-	u8 secret[skprefixlen + SECRET_LEN];
-	u8 * const sk = &secret[skprefixlen];
 	u8 seed[SEED_LEN];
 	u8 hashsrc[checksumstrlen + PUBLIC_LEN + 1];
 	u8 wpk[PUBLIC_LEN + 1];
@@ -318,19 +315,15 @@ static void *dofastwork(void *task)
 #endif
 	PREFILTER
 
-	memcpy(secret,skprefix,skprefixlen);
+	memcpy(secret,skprefix,SKPREFIX_SIZE);
 	wpk[PUBLIC_LEN] = 0;
 	memset(&pubonion,0,sizeof(pubonion));
-	memcpy(pubonion.raw,pkprefix,pkprefixlen);
+	memcpy(pubonion.raw,pkprefix,PKPREFIX_SIZE);
 	// write version later as it will be overwritten by hash
 	memcpy(hashsrc,checksumstr,checksumstrlen);
 	hashsrc[checksumstrlen + PUBLIC_LEN] = 0x03; // version
 
-	sname = (char *) malloc(workdirlen + ONIONLEN + 63 + 1);
+	sname = makesname();
-	if (!sname)
-		abort();
-	if (workdir)
-		memcpy(sname,workdir,workdirlen);
 
 initseed:
 #ifdef STATISTICS
@@ -410,7 +403,8 @@ static void printhelp(FILE *out,const char *progname)
 		"\t-f  - instead of specifying filter(s) via commandline, specify filter file which contains filters separated by newlines\n"
 		"\t-q  - do not print diagnostic output to stderr\n"
 		"\t-x  - do not print onion names\n"
-		"\t-o filename  - output onion names to specified file\n"
+		"\t-o filename  - output onion names to specified file (append)\n"
+		"\t-O filename  - output onion names to specified file (overwrite)\n"
 		"\t-F  - include directory names in onion names output\n"
 		"\t-d dirname  - output directory\n"
 		"\t-t numthreads  - specify number of threads (default - auto)\n"
@@ -422,6 +416,8 @@ static void printhelp(FILE *out,const char *progname)
 		"\t-s  - print statistics each 10 seconds\n"
 		"\t-S t  - print statistics every specified ammount of seconds\n"
 		"\t-T  - do not reset statistics counters when printing\n"
+		"\t-y  - output generated keys in yaml format instead of dumping them to filesystem\n"
+		"\t-Y [filename [host.onion]]  - parse yaml encoded input and extract key(s) to filesystem\n"
 		,progname,progname);
 	fflush(out);
 }
@@ -433,6 +429,7 @@ enum {
 	Q_FAILOPENOUTPUT,
 	Q_FAILTHREAD,
 	Q_FAILTIME,
+	Q_FAILOPENINPUT,
 } ;
 
 static void e_additional()
@@ -482,11 +479,15 @@ VEC_STRUCT(threadvec, pthread_t);
 int main(int argc,char **argv)
 {
 	const char *outfile = 0;
+	const char *infile = 0;
+	const char *hostname = 0;
 	const char *arg;
 	int ignoreargs = 0;
 	int dirnameflag = 0;
 	int numthreads = 0;
 	int fastkeygen = 1;
+	int yamlinput = 0;
+	int outfileoverwrite = 0;
 	struct threadvec threads;
 #ifdef STATISTICS
 	struct statsvec stats;
@@ -501,8 +502,6 @@ int main(int argc,char **argv)
 
 	setvbuf(stderr,0,_IONBF,0);
 	fout = stdout;
-	pthread_mutex_init(&keysgenerated_mutex,0);
-	pthread_mutex_init(&fout_mutex,0);
 
 	const char *progname = argv[0];
 	if (argc <= 1) {
@@ -556,6 +555,14 @@ int main(int argc,char **argv)
 			else if (*arg == 'x')
 				fout = 0;
 			else if (*arg == 'o') {
+				outfileoverwrite = 0;
+				if (argc--)
+					outfile = *argv++;
+				else
+					e_additional();
+			}
+			else if (*arg == 'O') {
+				outfileoverwrite = 1;
 				if (argc--)
 					outfile = *argv++;
 				else
@@ -615,6 +622,27 @@ int main(int argc,char **argv)
 				e_nostatistics();
 #endif
 			}
+			else if (*arg == 'y')
+				yamloutput = 1;
+			else if (*arg == 'Y') {
+				yamlinput = 1;
+				if (argc) {
+					--argc;
+					infile = *argv++;
+					if (!*infile)
+						infile = 0;
+					if (argc) {
+						--argc;
+						hostname = *argv++;
+						if (!*hostname)
+							hostname = 0;
+						if (hostname && strlen(hostname) != ONION_LEN) {
+							fprintf(stderr,"bad onion argument length\n");
+							exit(Q_UNRECOGNISED);
+						}
+					}
+				}
+			}
 			else {
 				fprintf(stderr,"unrecognised argument: -%c\n",*arg);
 				exit(Q_UNRECOGNISED);
@@ -627,13 +655,55 @@ int main(int argc,char **argv)
 	}
 
 	if (outfile) {
-		fout = fopen(outfile,"w");
+		fout = fopen(outfile,!outfileoverwrite ? "a" : "w");
 		if (!fout) {
 			perror("failed to open output file");
 			exit(Q_FAILOPENOUTPUT);
 		}
 	}
 
+	if (!fout && yamloutput) {
+		fprintf(stderr,"nil output with yaml mode does not make sense\n");
+		exit(Q_FAILOPENOUTPUT); // define new err code?
+	}
+
+	if (workdir)
+		createdir(workdir,1);
+
+	direndpos = workdirlen;
+	onionendpos = workdirlen + ONION_LEN;
+
+	if (!dirnameflag) {
+		printstartpos = direndpos;
+		printlen = ONION_LEN + 1; // + '\n'
+	} else {
+		printstartpos = 0;
+		printlen = onionendpos + 1; // + '\n'
+	}
+
+	if (yamlinput) {
+		char *sname = makesname();
+		FILE *fin = stdin;
+		if (infile) {
+			fin = fopen(infile,"r");
+			if (!fin) {
+				fprintf(stderr,"failed to open input file\n");
+				return Q_FAILOPENINPUT;
+			}
+		}
+		tret = yamlin_parseandcreate(fin,sname,hostname);
+		if (infile) {
+			fclose(fin);
+			fin = 0;
+		}
+		free(sname);
+
+		if (tret)
+			return tret;
+
+		goto done;
+	}
+
 	filters_prepare();
 
 	filters_print();
@@ -650,19 +720,11 @@ int main(int argc,char **argv)
 		fprintf(stderr,"WARNING: -N switch will produce bogus results because we can't know filter width. reconfigure with --enable-besort and recompile.\n");
 #endif
 
-	if (workdir)
+	if (yamloutput)
-		createdir(workdir,1);
+		yamlout_init();
 
-	direndpos = workdirlen;
+	pthread_mutex_init(&keysgenerated_mutex,0);
-	onionendpos = workdirlen + ONIONLEN;
+	pthread_mutex_init(&fout_mutex,0);
-
-	if (!dirnameflag) {
-		printstartpos = direndpos;
-		printlen = ONIONLEN + 1;
-	} else {
-		printstartpos = 0;
-		printlen = onionendpos + 1;
-	}
 
 	if (numthreads <= 0) {
 		numthreads = cpucount();
@@ -806,8 +868,13 @@ int main(int argc,char **argv)
 	if (!quietflag)
 		fprintf(stderr," done.\n");
 
+	if (yamloutput)
+		yamlout_clean();
+
 	pthread_mutex_destroy(&keysgenerated_mutex);
 	pthread_mutex_destroy(&fout_mutex);
+
+done:
 	filters_clean();
 
 	if (outfile)

test_base64.c

@@ -0,0 +1,48 @@
+#include <stddef.h>
+#include <stdint.h>
+#include "types.h"
+#include "base64.h"
+#include <string.h>
+#include <assert.h>
+#include <stdio.h>
+#include <sodium/randombytes.h>
+
+struct texttestcase {
+	const char *in;
+	const char *out;
+	const char *rev;
+} tests0[] = {
+	{"", "", ""},
+	{"f", "Zg==", "f"},
+	{"fo", "Zm8=", "fo"},
+	{"foo", "Zm9v", "foo"},
+	{"foob", "Zm9vYg==", "foob"},
+	{"fooba", "Zm9vYmE=", "fooba"},
+	{"foobar", "Zm9vYmFy", "foobar"},
+};
+
+int main(void)
+{
+	char buf[1024], buf2[1024];
+	size_t r;
+	for (size_t i = 0; i < sizeof(tests0)/sizeof(tests0[0]); ++i) {
+		base64_to(buf, (const u8 *)tests0[i].in, strlen(tests0[i].in));
+		if (strcmp(buf, tests0[i].out) != 0) {
+			printf("invalid encoding result: \"%s\" -> encoded as \"%s\", but expected \"%s\".\n",
+						 tests0[i].in, buf, tests0[i].out);
+			return 1;
+		}
+		if (!base64_valid(buf,0)) {
+			printf("encoded data is considered invalid\n");
+			return 3;
+		}
+		r = base64_from((u8 *)buf2, buf, strlen(buf));
+		buf2[r] = '\0';
+		if (strcmp(buf2, tests0[i].rev) != 0) {
+			printf("invalid decoding result: encoded \"%s\", decoded as \"%s\", but expected \"%s\".\n",
+						 tests0[i].out, buf2, tests0[i].rev);
+			return 2;
+		}
+	}
+	return 0;
+}

yaml.c

@@ -0,0 +1,268 @@
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <time.h>
+#include <pthread.h>
+
+#include "types.h"
+#include "yaml.h"
+#include "ioutil.h"
+#include "base64.h"
+#include "common.h"
+
+#define LINEFEED_LEN       (sizeof(char))
+#define NULLTERM_LEN       (sizeof(char))
+#define PATH_SEPARATOR_LEN (sizeof(char))
+
+static const char keys_field_generated[] = "---";
+static const char keys_field_hostname[]  = "hostname: ";
+static const char keys_field_publickey[] = "hs_ed25519_public_key: ";
+static const char keys_field_secretkey[] = "hs_ed25519_secret_key: ";
+static const char keys_field_time[]      = "time: ";
+
+#define KEYS_FIELD_GENERATED_LEN (sizeof(keys_field_generated) - NULLTERM_LEN)
+#define KEYS_FIELD_HOSTNAME_LEN  (sizeof(keys_field_hostname)  - NULLTERM_LEN)
+#define KEYS_FIELD_PUBLICKEY_LEN (sizeof(keys_field_publickey) - NULLTERM_LEN)
+#define KEYS_FIELD_SECRETKEY_LEN (sizeof(keys_field_secretkey) - NULLTERM_LEN)
+#define KEYS_FIELD_TIME_LEN      (sizeof(keys_field_time)      - NULLTERM_LEN)
+
+static const char hostname_example[] = "xxxxxvsjzke274nisktdqcl3eqm5ve3m6iur6vwme7m5p6kxivrvjnyd.onion";
+static const char pubkey_example[]   = "PT0gZWQyNTUxOXYxLXB1YmxpYzogdHlwZTAgPT0AAAC973vWScqJr/GokqY4CXskGdqTbPIpH1bMJ9nX+VdFYw==";
+static const char seckey_example[]   = "PT0gZWQyNTUxOXYxLXNlY3JldDogdHlwZTAgPT0AAACwCPMr6rvBRtkW7ZzZ8P7Ne4acRZrhPrN/EF6AETRraFGvdrkW5es4WXB2UxrbuUf8zPoIKkXK5cpdakYdUeM3";
+static const char time_example[]     = "2018-07-04 21:31:20 Z";
+
+#define HOSTNAME_LEN (sizeof(hostname_example) - NULLTERM_LEN)
+#define PUBKEY_LEN   (sizeof(pubkey_example)   - NULLTERM_LEN)
+#define SECKEY_LEN   (sizeof(seckey_example)   - NULLTERM_LEN)
+#define TIME_LEN     (sizeof(time_example)     - NULLTERM_LEN)
+
+#define KEYS_LEN ( \
+	KEYS_FIELD_GENERATED_LEN + LINEFEED_LEN + \
+	KEYS_FIELD_HOSTNAME_LEN + HOSTNAME_LEN + LINEFEED_LEN + \
+	KEYS_FIELD_PUBLICKEY_LEN + PUBKEY_LEN + LINEFEED_LEN + \
+	KEYS_FIELD_SECRETKEY_LEN + SECKEY_LEN + LINEFEED_LEN + \
+	KEYS_FIELD_TIME_LEN + TIME_LEN + LINEFEED_LEN \
+)
+
+static pthread_mutex_t tminfo_mutex;
+
+void yamlout_init()
+{
+	pthread_mutex_init(&tminfo_mutex,0);
+}
+
+void yamlout_clean()
+{
+	pthread_mutex_destroy(&tminfo_mutex);
+}
+
+#define BUF_APPEND(buf,offset,src,srclen) \
+do { \
+	memcpy(&buf[offset],(src),(srclen)); \
+	offset += (srclen); \
+} while (0)
+#define BUF_APPEND_CSTR(buf,offset,src) BUF_APPEND(buf,offset,src,strlen(src))
+#define BUF_APPEND_CHAR(buf,offset,c) buf[offset++] = (c)
+
+void yamlout_writekeys(const char *hostname,const u8 *formated_public,const u8 *formated_secret)
+{
+	char keysbuf[KEYS_LEN];
+	char pubkeybuf[PUBKEY_LEN + NULLTERM_LEN];
+	char seckeybuf[SECKEY_LEN + NULLTERM_LEN];
+	char timebuf[TIME_LEN + NULLTERM_LEN];
+	size_t offset = 0;
+
+	BUF_APPEND(keysbuf,offset,keys_field_generated,KEYS_FIELD_GENERATED_LEN);
+	BUF_APPEND_CHAR(keysbuf,offset,'\n');
+
+	BUF_APPEND(keysbuf,offset,keys_field_hostname,KEYS_FIELD_HOSTNAME_LEN);
+	BUF_APPEND(keysbuf,offset,hostname,ONION_LEN);
+	BUF_APPEND_CHAR(keysbuf,offset,'\n');
+
+	BUF_APPEND(keysbuf,offset,keys_field_publickey,KEYS_FIELD_PUBLICKEY_LEN);
+	base64_to(pubkeybuf,formated_public,FORMATTED_PUBLIC_LEN);
+	BUF_APPEND(keysbuf,offset,pubkeybuf,PUBKEY_LEN);
+	BUF_APPEND_CHAR(keysbuf,offset,'\n');
+
+	BUF_APPEND(keysbuf,offset,keys_field_secretkey,KEYS_FIELD_SECRETKEY_LEN);
+	base64_to(seckeybuf,formated_secret,FORMATTED_SECRET_LEN);
+	BUF_APPEND(keysbuf,offset,seckeybuf,SECKEY_LEN);
+	BUF_APPEND_CHAR(keysbuf,offset,'\n');
+
+	BUF_APPEND(keysbuf,offset,keys_field_time,KEYS_FIELD_TIME_LEN);
+
+	time_t currtime;
+	time(&currtime);
+	struct tm *tm_info;
+
+	pthread_mutex_lock(&tminfo_mutex);
+	tm_info = gmtime(&currtime);
+	strftime(timebuf,sizeof(timebuf),"%Y-%m-%d %H:%M:%S Z",tm_info);
+	pthread_mutex_unlock(&tminfo_mutex);
+
+	BUF_APPEND(keysbuf,offset,timebuf,TIME_LEN);
+	BUF_APPEND_CHAR(keysbuf,offset,'\n');
+
+	assert(offset == KEYS_LEN);
+
+	pthread_mutex_lock(&fout_mutex);
+	fwrite(keysbuf,sizeof(keysbuf),1,fout);
+	fflush(fout);
+	pthread_mutex_unlock(&fout_mutex);
+}
+
+#undef BUF_APPEND_CHAR
+#undef BUF_APPEND_CSTR
+#undef BUF_APPEND
+
+// pseudo YAML parser
+int yamlin_parseandcreate(FILE *fin,char *sname,const char *hostname)
+{
+	char line[256];
+	size_t len;
+	u8 pubbuf[FORMATTED_PUBLIC_LEN];
+	u8 secbuf[FORMATTED_SECRET_LEN];
+	int hashost = 0,haspub = 0,hassec = 0,skipthis = 0;
+	enum keytype { HOST, PUB, SEC } keyt;
+
+	while (!feof(fin) && !ferror(fin)) {
+		if (!fgets(line,sizeof(line),fin))
+			break;
+		
+		len = strlen(line);
+		
+		// trim whitespace from the end
+		while (len != 0 && (line[len-1] == ' ' || line[len-1] == '\n' || line[len-1] == '\r'))
+			line[--len] = '\0';
+		
+		// skip empty lines
+		if (len == 0)
+			continue;
+		
+		if (len >= 3 && line[0] == '-' && line[1] == '-' && line[2] == '-') {
+			// end of document indicator
+			if (!skipthis && (hashost || haspub || hassec)) {
+				fprintf(stderr,"ERROR: incomplete record\n");
+				return 1;
+			}
+			hashost = haspub = hassec = skipthis = 0;
+			continue;
+		}
+		
+		if (skipthis)
+			continue;
+		
+		char *start = line;
+		// trim whitespace
+		while (len != 0 && *start == ' ') {
+			++start;
+			--len;
+		}
+		// find ':'
+		char *p = start;
+		for (;*p != '\0';++p) {
+			if (*p == ':') {
+				*p++ = '\0';
+				goto foundkey;
+			}
+		}
+		// not `key: value`
+		fprintf(stderr,"ERROR: invalid syntax\n");
+		return 1; // XXX could continue too there but eh
+
+	foundkey:
+
+		if (!strcmp(start,"hostname"))
+			keyt = HOST;
+		else if (!strcmp(start,"hs_ed25519_public_key"))
+			keyt = PUB;
+		else if (!strcmp(start,"hs_ed25519_secret_key"))
+			keyt = SEC;
+		else
+			continue; // uninterested
+
+		// skip WS
+		while (*p == ' ')
+			++p;
+		if (*p == '!') {
+			// skip ! tag
+			while (*p != '\0' && *p != ' ')
+				++p;
+			// skip WS
+			while (*p == ' ')
+				++p;
+		}
+		len = strlen(p);
+		switch (keyt) {
+			case HOST:
+				if (len != ONION_LEN) {
+					fprintf(stderr,"ERROR: invalid hostname syntax\n");
+					return 1;
+				}
+				if (!hostname || !strcmp(hostname,p)) {
+					memcpy(&sname[direndpos],p,len + 1);
+					hashost = 1;
+				} else
+					skipthis = 1;
+				break;
+			case PUB:
+				if (len != PUBKEY_LEN || !base64_valid(p,0)) {
+					fprintf(stderr,"ERROR: invalid pubkey syntax\n");
+					return 1;
+				}
+				base64_from(pubbuf,p,len);
+				haspub = 1;
+				break;
+			case SEC:
+				if (len != SECKEY_LEN || !base64_valid(p,0)) {
+					fprintf(stderr,"ERROR: invalid seckey syntax\n");
+					return 1;
+				}
+				base64_from(secbuf,p,len);
+				hassec = 1;
+				break;
+		}
+		if (hashost && haspub && hassec) {
+			if (createdir(sname,1) != 0) {
+				fprintf(stderr,"ERROR: could not create directory for key output\n");
+				return 1;
+			}
+
+			strcpy(&sname[onionendpos],"/hs_ed25519_secret_key");
+			writetofile(sname,secbuf,FORMATTED_SECRET_LEN,1);
+
+			strcpy(&sname[onionendpos],"/hs_ed25519_public_key");
+			writetofile(sname,pubbuf,FORMATTED_PUBLIC_LEN,0);
+
+			strcpy(&sname[onionendpos],"/hostname");
+			FILE *hfile = fopen(sname,"w");
+			sname[onionendpos] = '\n';
+			if (hfile) {
+				fwrite(&sname[direndpos],ONION_LEN + 1,1,hfile);
+				fclose(hfile);
+			}
+			if (fout) {
+				fwrite(&sname[printstartpos],printlen,1,fout);
+				fflush(fout);
+			}
+			if (hostname)
+				return 0; // finished
+			skipthis = 1;
+		}
+	}
+	
+	if (!feof(fin)) {
+		fprintf(stderr,"error while reading input\n");
+		return 1;
+	}
+	
+	if (hostname) {
+		fprintf(stderr,"hostname wasn't found in input\n");
+		return 1;
+	}
+	
+	return 0;
+}

yaml.h

@@ -0,0 +1,4 @@
+extern void yamlout_init();
+extern void yamlout_clean();
+extern void yamlout_writekeys(const char *hostname,const u8 *formated_public,const u8 *formated_secret);
+extern int yamlin_parseandcreate(FILE *fin,char *sname,const char *hostname);