[PATCH] fio: Add option for using crc32 for verifies.

It's a lot faster, but md5 is a lot less likely to collide.

diff --git a/Makefile b/Makefile
index a39fdf62b48fe4fdd2f6273903015d7158598d52..dc6e1205015b7d8073d588db21761094f6963905 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@ all: depend $(PROGS) $(SCRIPTS)
 dops: dops.o
        $(CC) $(CFLAGS) -o $@ $(filter %.o,$^) -laio
 
-fio: fio.o fio-ini.o md5.o
+fio: fio.o fio-ini.o md5.o crc32.o
        $(CC) $(CFLAGS) -o $@ $(filter %.o,$^) -lpthread -laio -lm
 
 sgioread: sgioread.o

diff --git a/README.fio b/README.fio
index f789bced378c56dda31caaf5925a08ed2aa425d9..a353efff3e6815c7faf7697a0b1898ab47351ed2 100644 (file)
--- a/README.fio
+++ b/README.fio
@@ -53,7 +53,8 @@ The <jobs> format is as follows:
        create_serialize=x      If 'x', serialize file creation.
        create_fsync=x  If 'x', run fsync() after file creation.
        loops=x         Run the job 'x' number of times.
-       verify=x        If 'x' and writing, verify data written.
+       verify=x        If 'x' == md5, use md5 for verifies. If 'x' == crc32,
+                       use crc32 for verifies.
        stonewall       Wait for preceeding jobs to end before running.
        numjobs=x       Create 'x' similar entries for this job
        mmap=x          If 'x', use mmap for data transfers

diff --git a/crc32.c b/crc32.c
new file mode 100644 (file)
index 0000000..ba6cc06
--- /dev/null
+++ b/crc32.c
@@ -0,0 +1,84 @@
+/* crc32 -- calculate and POSIX.2 checksum 
+   Copyright (C) 92, 1995-1999 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software Foundation,
+   Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */
+
+#include "crc32.h"
+
+static const unsigned long crctab[256] = {
+  0x0,
+  0x04C11DB7, 0x09823B6E, 0x0D4326D9, 0x130476DC, 0x17C56B6B,
+  0x1A864DB2, 0x1E475005, 0x2608EDB8, 0x22C9F00F, 0x2F8AD6D6,
+  0x2B4BCB61, 0x350C9B64, 0x31CD86D3, 0x3C8EA00A, 0x384FBDBD,
+  0x4C11DB70, 0x48D0C6C7, 0x4593E01E, 0x4152FDA9, 0x5F15ADAC,
+  0x5BD4B01B, 0x569796C2, 0x52568B75, 0x6A1936C8, 0x6ED82B7F,
+  0x639B0DA6, 0x675A1011, 0x791D4014, 0x7DDC5DA3, 0x709F7B7A,
+  0x745E66CD, 0x9823B6E0, 0x9CE2AB57, 0x91A18D8E, 0x95609039,
+  0x8B27C03C, 0x8FE6DD8B, 0x82A5FB52, 0x8664E6E5, 0xBE2B5B58,
+  0xBAEA46EF, 0xB7A96036, 0xB3687D81, 0xAD2F2D84, 0xA9EE3033,
+  0xA4AD16EA, 0xA06C0B5D, 0xD4326D90, 0xD0F37027, 0xDDB056FE,
+  0xD9714B49, 0xC7361B4C, 0xC3F706FB, 0xCEB42022, 0xCA753D95,
+  0xF23A8028, 0xF6FB9D9F, 0xFBB8BB46, 0xFF79A6F1, 0xE13EF6F4,
+  0xE5FFEB43, 0xE8BCCD9A, 0xEC7DD02D, 0x34867077, 0x30476DC0,
+  0x3D044B19, 0x39C556AE, 0x278206AB, 0x23431B1C, 0x2E003DC5,
+  0x2AC12072, 0x128E9DCF, 0x164F8078, 0x1B0CA6A1, 0x1FCDBB16,
+  0x018AEB13, 0x054BF6A4, 0x0808D07D, 0x0CC9CDCA, 0x7897AB07,
+  0x7C56B6B0, 0x71159069, 0x75D48DDE, 0x6B93DDDB, 0x6F52C06C,
+  0x6211E6B5, 0x66D0FB02, 0x5E9F46BF, 0x5A5E5B08, 0x571D7DD1,
+  0x53DC6066, 0x4D9B3063, 0x495A2DD4, 0x44190B0D, 0x40D816BA,
+  0xACA5C697, 0xA864DB20, 0xA527FDF9, 0xA1E6E04E, 0xBFA1B04B,
+  0xBB60ADFC, 0xB6238B25, 0xB2E29692, 0x8AAD2B2F, 0x8E6C3698,
+  0x832F1041, 0x87EE0DF6, 0x99A95DF3, 0x9D684044, 0x902B669D,
+  0x94EA7B2A, 0xE0B41DE7, 0xE4750050, 0xE9362689, 0xEDF73B3E,
+  0xF3B06B3B, 0xF771768C, 0xFA325055, 0xFEF34DE2, 0xC6BCF05F,
+  0xC27DEDE8, 0xCF3ECB31, 0xCBFFD686, 0xD5B88683, 0xD1799B34,
+  0xDC3ABDED, 0xD8FBA05A, 0x690CE0EE, 0x6DCDFD59, 0x608EDB80,
+  0x644FC637, 0x7A089632, 0x7EC98B85, 0x738AAD5C, 0x774BB0EB,
+  0x4F040D56, 0x4BC510E1, 0x46863638, 0x42472B8F, 0x5C007B8A,
+  0x58C1663D, 0x558240E4, 0x51435D53, 0x251D3B9E, 0x21DC2629,
+  0x2C9F00F0, 0x285E1D47, 0x36194D42, 0x32D850F5, 0x3F9B762C,
+  0x3B5A6B9B, 0x0315D626, 0x07D4CB91, 0x0A97ED48, 0x0E56F0FF,
+  0x1011A0FA, 0x14D0BD4D, 0x19939B94, 0x1D528623, 0xF12F560E,
+  0xF5EE4BB9, 0xF8AD6D60, 0xFC6C70D7, 0xE22B20D2, 0xE6EA3D65,
+  0xEBA91BBC, 0xEF68060B, 0xD727BBB6, 0xD3E6A601, 0xDEA580D8,
+  0xDA649D6F, 0xC423CD6A, 0xC0E2D0DD, 0xCDA1F604, 0xC960EBB3,
+  0xBD3E8D7E, 0xB9FF90C9, 0xB4BCB610, 0xB07DABA7, 0xAE3AFBA2,
+  0xAAFBE615, 0xA7B8C0CC, 0xA379DD7B, 0x9B3660C6, 0x9FF77D71,
+  0x92B45BA8, 0x9675461F, 0x8832161A, 0x8CF30BAD, 0x81B02D74,
+  0x857130C3, 0x5D8A9099, 0x594B8D2E, 0x5408ABF7, 0x50C9B640,
+  0x4E8EE645, 0x4A4FFBF2, 0x470CDD2B, 0x43CDC09C, 0x7B827D21,
+  0x7F436096, 0x7200464F, 0x76C15BF8, 0x68860BFD, 0x6C47164A,
+  0x61043093, 0x65C52D24, 0x119B4BE9, 0x155A565E, 0x18197087,
+  0x1CD86D30, 0x029F3D35, 0x065E2082, 0x0B1D065B, 0x0FDC1BEC,
+  0x3793A651, 0x3352BBE6, 0x3E119D3F, 0x3AD08088, 0x2497D08D,
+  0x2056CD3A, 0x2D15EBE3, 0x29D4F654, 0xC5A92679, 0xC1683BCE,
+  0xCC2B1D17, 0xC8EA00A0, 0xD6AD50A5, 0xD26C4D12, 0xDF2F6BCB,
+  0xDBEE767C, 0xE3A1CBC1, 0xE760D676, 0xEA23F0AF, 0xEEE2ED18,
+  0xF0A5BD1D, 0xF464A0AA, 0xF9278673, 0xFDE69BC4, 0x89B8FD09,
+  0x8D79E0BE, 0x803AC667, 0x84FBDBD0, 0x9ABC8BD5, 0x9E7D9662,
+  0x933EB0BB, 0x97FFAD0C, 0xAFB010B1, 0xAB710D06, 0xA6322BDF,
+  0xA2F33668, 0xBCB4666D, 0xB8757BDA, 0xB5365D03, 0xB1F740B4
+};
+
+unsigned long crc32(const void *buffer, unsigned long length)
+{
+       const unsigned char *cp = (const unsigned char *) buffer;
+       unsigned long crc = 0;
+
+       while (length--)
+               crc = (crc << 8) ^ crctab[((crc >> 24) ^ *(cp++)) & 0xFF];
+
+       return crc;
+}

diff --git a/crc32.h b/crc32.h
new file mode 100644 (file)
index 0000000..b7e5eee
--- /dev/null
+++ b/crc32.h
@@ -0,0 +1,23 @@
+/* crc32 -- calculate and POSIX.2 checksum 
+   Copyright (C) 92, 1995-1999 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software Foundation,
+   Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */
+
+#ifndef CRC32_H
+#define CRC32_H
+
+extern unsigned long crc32(const void * const, unsigned long);
+
+#endif

diff --git a/fio-ini.c b/fio-ini.c
index 4a8f4749b4fb10ec99f7b9994df840e73d8718eb..09c0eb3321b385d747f627aa21df3cc980043711 100644 (file)
--- a/fio-ini.c
+++ b/fio-ini.c
@@ -554,10 +554,6 @@ int parse_jobs_ini(char *file)
                                fgetpos(f, &off);
                                continue;
                        }
-                       if (!check_int(p, "verify", &td->verify)) {
-                               fgetpos(f, &off);
-                               continue;
-                       }
                        if (!check_int(p, "numjobs", &td->numjobs)) {
                                fgetpos(f, &off);
                                continue;
@@ -613,6 +609,16 @@ int parse_jobs_ini(char *file)
                                fgetpos(f, &off);
                                continue;
                        }
+                       if (!check_str(p, "verify", "md5")) {
+                               td->verify = VERIFY_MD5;
+                               fgetpos(f, &off);
+                               continue;
+                       }
+                       if (!check_str(p, "verify", "crc32")) {
+                               td->verify = VERIFY_CRC32;
+                               fgetpos(f, &off);
+                               continue;
+                       }
                        if (!check_strset(p, "sequential")) {
                                td->sequential = 1;
                                fgetpos(f, &off);

diff --git a/fio.c b/fio.c
index 976e0a6284afa5bd963cfcf2297d84f5f6d53d68..a840b6048ff9d59652c5d80a31461783c067ad74 100644 (file)
--- a/fio.c
+++ b/fio.c
@@ -449,16 +449,23 @@ static void hexdump(void *buffer, int len)
        printf("\n");
 }
 
-static int verify_io_u(struct io_u *io_u)
+static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u)
+{
+       unsigned char *p = (unsigned char *) io_u->buf;
+       unsigned long c;
+
+       p += sizeof(*hdr);
+       c = crc32(p, hdr->len - sizeof(*hdr));
+
+       return c != hdr->crc32;
+}
+
+static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u)
 {
-       struct verify_header *hdr = (struct verify_header *) io_u->buf;
        unsigned char *p = (unsigned char *) io_u->buf;
        struct md5_ctx md5_ctx;
        int ret;
 
-       if (hdr->fio_magic != FIO_HDR_MAGIC)
-               return 1;
-
        memset(&md5_ctx, 0, sizeof(md5_ctx));
        p += sizeof(*hdr);
        md5_update(&md5_ctx, p, hdr->len - sizeof(*hdr));
@@ -472,24 +479,62 @@ static int verify_io_u(struct io_u *io_u)
        return ret;
 }
 
+static int verify_io_u(struct io_u *io_u)
+{
+       struct verify_header *hdr = (struct verify_header *) io_u->buf;
+       int ret;
+
+       if (hdr->fio_magic != FIO_HDR_MAGIC)
+               return 1;
+
+       if (hdr->verify_type == VERIFY_MD5)
+               ret = verify_io_u_md5(hdr, io_u);
+       else if (hdr->verify_type == VERIFY_CRC32)
+               ret = verify_io_u_crc32(hdr, io_u);
+       else {
+               fprintf(stderr, "Bad verify type %d\n", hdr->verify_type);
+               ret = 1;
+       }
+
+       return ret;
+}
+
+static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
+{
+       hdr->crc32 = crc32(p, len);
+}
+
+static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
+{
+       struct md5_ctx md5_ctx;
+
+       memset(&md5_ctx, 0, sizeof(md5_ctx));
+       md5_update(&md5_ctx, p, len);
+       memcpy(hdr->md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash));
+}
+
 /*
  * fill body of io_u->buf with random data and add a header with the
  * (eg) sha1sum of that data.
  */
 static void populate_io_u(struct thread_data *td, struct io_u *io_u)
 {
-       struct md5_ctx md5_ctx;
-       struct verify_header hdr;
        unsigned char *p = (unsigned char *) io_u->buf;
+       struct verify_header hdr;
 
        hdr.fio_magic = FIO_HDR_MAGIC;
        hdr.len = io_u->buflen;
        p += sizeof(hdr);
        fill_random_bytes(td, p, io_u->buflen - sizeof(hdr));
 
-       memset(&md5_ctx, 0, sizeof(md5_ctx));
-       md5_update(&md5_ctx, p, io_u->buflen - sizeof(hdr));
-       memcpy(hdr.md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash));
+       if (td->verify == VERIFY_MD5) {
+               fill_md5(&hdr, p, io_u->buflen - sizeof(hdr));
+               hdr.verify_type = VERIFY_MD5;
+       } else {
+               fill_crc32(&hdr, p, io_u->buflen - sizeof(hdr));
+               hdr.verify_type = VERIFY_CRC32;
+       }
+
        memcpy(io_u->buf, &hdr, sizeof(hdr));
 }
 
@@ -543,7 +588,7 @@ static struct io_u *get_io_u(struct thread_data *td)
 
        td->last_bytes += io_u->buflen;
 
-       if (td->verify)
+       if (td->verify != VERIFY_NONE)
                populate_io_u(td, io_u);
 
        if (td->use_aio) {
@@ -1317,7 +1362,7 @@ static int setup_file_mmap(struct thread_data *td)
        else {
                flags = PROT_WRITE;
 
-               if (td->verify)
+               if (td->verify != VERIFY_NONE)
                        flags |= PROT_READ;
        }
 
@@ -1508,7 +1553,7 @@ static void *thread_main(void *data)
                td->runtime += mtime_since_now(&td->start);
                update_rusage_stat(td);
 
-               if (!td->verify)
+               if (td->verify == VERIFY_NONE)
                        continue;
 
                clear_io_state(td);

diff --git a/fio.h b/fio.h
index b8d39670e196d91d3619f72a41438885d37fd621..96df3bf1149cdd5f2b2284bbb7787ab4b2b07e40 100644 (file)
--- a/fio.h
+++ b/fio.h
@@ -10,6 +10,7 @@
 
 #include "list.h"
 #include "md5.h"
+#include "crc32.h"
 #include "arch.h"
 
 struct io_stat {
@@ -39,10 +40,20 @@ struct io_piece {
 
 #define FIO_HDR_MAGIC  0xf00baaef
 
+enum {
+       VERIFY_NONE = 0,
+       VERIFY_MD5,
+       VERIFY_CRC32,
+};
+
 struct verify_header {
        unsigned int fio_magic;
        unsigned int len;
-       char md5_digest[MD5_HASH_WORDS * 4];
+       unsigned int verify_type;
+       union {
+               char md5_digest[MD5_HASH_WORDS * 4];
+               unsigned long crc32;
+       };
 };
 
 struct group_run_stats {