[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[PATCH] base64: improve encoding I/O efficiency
From: |
Pádraig Brady |
Subject: |
[PATCH] base64: improve encoding I/O efficiency |
Date: |
Mon, 11 Nov 2013 17:16:38 +0000 |
Replace fputs() calls with fputc() to improve performance.
Since the I/O overhead is significant to the relatively
simple processing done by this utility, use the simpler
function to output '\n'.
Time to process a 100MiB file was measured to
decrease from 0.417s to 0.383s, i.e. an 8% improvement.
Related to these changes, is a processing improvement in
gnulib, which increases throughput by 60% when processing
full buffers, which improves processing of a 100MiB file
with standard wrapped output, down to 0.256s.
http://lists.gnu.org/archive/html/bug-gnulib/2013-11/msg00057.html
Also increase the encoding buffer size from 3 to 30KiB.
This was seen to give a further 8% improvement, taking
processing time down to 0.235s is the wrapped output case.
The decoding size buffer is not adjusted,
due to the noted caveat with --ignore-garbage.
* src/base64.c (BLOCKSIZE): Split into ENC_ and DEC_ variants,
with the former increased from 3KiB to 30KiB.
(wrap_write): Use the simpler fputc() rather than fputs()
to output the '\n' character. Also check against EOF
rather than < 0 for errors.
(do_encode): Likewise.
* NEWS: Mention the large increase in performance, which
with the I/O improvements in coreutils and the processing
improvement in gnulib, amount to about a 60% throughput increase.
---
NEWS | 2 ++
src/base64.c | 40 +++++++++++++++++++++-------------------
2 files changed, 23 insertions(+), 19 deletions(-)
diff --git a/NEWS b/NEWS
index 7028c0d..fafe06d 100644
--- a/NEWS
+++ b/NEWS
@@ -88,6 +88,8 @@ GNU coreutils NEWS -*-
outline -*-
** Improvements
+ base64 encoding throughtput for bulk data is increased by about 60%.
+
stat and tail work better with EFIVARFS, EXOFS, F2FS, SNFS and UBIFS.
stat -f --format=%T now reports the file system type, and tail -f now uses
inotify for files on those file systems, rather than the default (for unknown
diff --git a/src/base64.c b/src/base64.c
index 0a40068..d1bdf3e 100644
--- a/src/base64.c
+++ b/src/base64.c
@@ -30,6 +30,7 @@
#include "xstrtol.h"
#include "quote.h"
#include "quotearg.h"
+#include "verify.h"
#include "xfreopen.h"
#include "base64.h"
@@ -89,15 +90,16 @@ from any other non-alphabet bytes in the encoded
stream.\n"),
exit (status);
}
+#define ENC_BLOCKSIZE (1024*3*10)
+#define ENC_B64BLOCKSIZE BASE64_LENGTH (ENC_BLOCKSIZE)
/* Note that increasing this may decrease performance if --ignore-garbage
- is used, because of the memmove operation below. */
-#define BLOCKSIZE 3072
-#define B64BLOCKSIZE BASE64_LENGTH (BLOCKSIZE)
+ is used, because of the memmove operation below. */
+#define DEC_BLOCKSIZE (1024*3)
+#define DEC_B64BLOCKSIZE BASE64_LENGTH (DEC_BLOCKSIZE)
/* Ensure that BLOCKSIZE is a multiple of 3 and 4. */
-#if BLOCKSIZE % 12 != 0
-# error "invalid BLOCKSIZE"
-#endif
+verify (ENC_BLOCKSIZE % 12 == 0);
+verify (DEC_BLOCKSIZE % 12 == 0);
static void
wrap_write (const char *buffer, size_t len,
@@ -120,7 +122,7 @@ wrap_write (const char *buffer, size_t len,
if (to_write == 0)
{
- if (fputs ("\n", out) < 0)
+ if (fputc ('\n', out) == EOF)
error (EXIT_FAILURE, errno, _("write error"));
*current_column = 0;
}
@@ -138,8 +140,8 @@ static void
do_encode (FILE *in, FILE *out, uintmax_t wrap_column)
{
size_t current_column = 0;
- char inbuf[BLOCKSIZE];
- char outbuf[B64BLOCKSIZE];
+ char inbuf[ENC_BLOCKSIZE];
+ char outbuf[ENC_B64BLOCKSIZE];
size_t sum;
do
@@ -149,14 +151,14 @@ do_encode (FILE *in, FILE *out, uintmax_t wrap_column)
sum = 0;
do
{
- n = fread (inbuf + sum, 1, BLOCKSIZE - sum, in);
+ n = fread (inbuf + sum, 1, ENC_BLOCKSIZE - sum, in);
sum += n;
}
- while (!feof (in) && !ferror (in) && sum < BLOCKSIZE);
+ while (!feof (in) && !ferror (in) && sum < ENC_BLOCKSIZE);
if (sum > 0)
{
- /* Process input one block at a time. Note that BLOCKSIZE %
+ /* Process input one block at a time. Note that ENC_BLOCKSIZE %
3 == 0, so that no base64 pads will appear in output. */
base64_encode (inbuf, sum, outbuf, BASE64_LENGTH (sum));
@@ -164,10 +166,10 @@ do_encode (FILE *in, FILE *out, uintmax_t wrap_column)
¤t_column, out);
}
}
- while (!feof (in) && !ferror (in) && sum == BLOCKSIZE);
+ while (!feof (in) && !ferror (in) && sum == ENC_BLOCKSIZE);
/* When wrapping, terminate last line. */
- if (wrap_column && current_column > 0 && fputs ("\n", out) < 0)
+ if (wrap_column && current_column > 0 && fputc ('\n', out) == EOF)
error (EXIT_FAILURE, errno, _("write error"));
if (ferror (in))
@@ -177,8 +179,8 @@ do_encode (FILE *in, FILE *out, uintmax_t wrap_column)
static void
do_decode (FILE *in, FILE *out, bool ignore_garbage)
{
- char inbuf[B64BLOCKSIZE];
- char outbuf[BLOCKSIZE];
+ char inbuf[DEC_B64BLOCKSIZE];
+ char outbuf[DEC_BLOCKSIZE];
size_t sum;
struct base64_decode_context ctx;
@@ -193,7 +195,7 @@ do_decode (FILE *in, FILE *out, bool ignore_garbage)
sum = 0;
do
{
- n = fread (inbuf + sum, 1, B64BLOCKSIZE - sum, in);
+ n = fread (inbuf + sum, 1, DEC_B64BLOCKSIZE - sum, in);
if (ignore_garbage)
{
@@ -210,7 +212,7 @@ do_decode (FILE *in, FILE *out, bool ignore_garbage)
if (ferror (in))
error (EXIT_FAILURE, errno, _("read error"));
}
- while (sum < B64BLOCKSIZE && !feof (in));
+ while (sum < DEC_B64BLOCKSIZE && !feof (in));
/* The following "loop" is usually iterated just once.
However, when it processes the final input buffer, we want
@@ -220,7 +222,7 @@ do_decode (FILE *in, FILE *out, bool ignore_garbage)
{
if (k == 1 && ctx.i == 0)
break;
- n = BLOCKSIZE;
+ n = DEC_BLOCKSIZE;
ok = base64_decode_ctx (&ctx, inbuf, (k == 0 ? sum : 0), outbuf, &n);
if (fwrite (outbuf, 1, n, out) < n)
--
1.7.7.6
- [PATCH] base64: improve encoding I/O efficiency,
Pádraig Brady <=