X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=verify.c;h=10bad7e2bab5f65c8b9168333a5f2567e40d19e1;hp=afb4cb7d4afbda3a896c34ab4da805c52238a26e;hb=e92d3d714acfe180413f63a5e5327de594aad0ef;hpb=bac39e0e8807a1d52863ad8304e67221df2bc63b

diff --git a/verify.c b/verify.c
index afb4cb7d..10bad7e2 100644
--- a/verify.c
+++ b/verify.c
@@ -5,8 +5,11 @@
 #include <fcntl.h>
 #include <string.h>
 #include <assert.h>
+#include <pthread.h>
 
 #include "fio.h"
+#include "verify.h"
+#include "smalloc.h"
 
 #include "crc/md5.h"
 #include "crc/crc64.h"
@@ -128,6 +131,7 @@ static inline unsigned int __hdr_size(int verify_type)
 		break;
 	case VERIFY_CRC32C:
 	case VERIFY_CRC32:
+	case VERIFY_CRC32C_INTEL:
 		len = sizeof(struct vhdr_crc32);
 		break;
 	case VERIFY_CRC16:
@@ -184,7 +188,7 @@ static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td,
 	if (vh->offset != io_u->offset + header_num * td->o.verify_interval) {
 		log_err("meta: verify failed at %llu/%u\n",
 				io_u->offset + header_num * hdr->len, hdr->len);
-		return EIO;
+		return EILSEQ;
 	}
 
 	return 0;
@@ -210,7 +214,7 @@ static int verify_io_u_sha512(struct verify_header *hdr, struct io_u *io_u,
 				io_u->offset + header_num * hdr->len, hdr->len);
 		hexdump(vh->sha512, sizeof(vh->sha512));
 		hexdump(sha512_ctx.buf, sizeof(sha512));
-		return EIO;
+		return EILSEQ;
 	}
 
 	return 0;
@@ -236,7 +240,7 @@ static int verify_io_u_sha256(struct verify_header *hdr, struct io_u *io_u,
 				io_u->offset + header_num * hdr->len, hdr->len);
 		hexdump(vh->sha256, sizeof(vh->sha256));
 		hexdump(sha256_ctx.buf, sizeof(sha256));
-		return EIO;
+		return EILSEQ;
 	}
 
 	return 0;
@@ -257,7 +261,7 @@ static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u,
 		log_err("crc7: verify failed at %llu/%u\n",
 				io_u->offset + header_num * hdr->len, hdr->len);
 		log_err("crc7: wanted %x, got %x\n", vh->crc7, c);
-		return EIO;
+		return EILSEQ;
 	}
 
 	return 0;
@@ -278,7 +282,7 @@ static int verify_io_u_crc16(struct verify_header *hdr, struct io_u *io_u,
 		log_err("crc16: verify failed at %llu/%u\n",
 				io_u->offset + header_num * hdr->len, hdr->len);
 		log_err("crc16: wanted %x, got %x\n", vh->crc16, c);
-		return EIO;
+		return EILSEQ;
 	}
 
 	return 0;
@@ -301,7 +305,7 @@ static int verify_io_u_crc64(struct verify_header *hdr, struct io_u *io_u,
 				hdr->len);
 		log_err("crc64: wanted %llx, got %llx\n",
 					(unsigned long long) vh->crc64, c);
-		return EIO;
+		return EILSEQ;
 	}
 
 	return 0;
@@ -322,7 +326,7 @@ static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u,
 		log_err("crc32: verify failed at %llu/%u\n",
 				io_u->offset + header_num * hdr->len, hdr->len);
 		log_err("crc32: wanted %x, got %x\n", vh->crc32, c);
-		return EIO;
+		return EILSEQ;
 	}
 
 	return 0;
@@ -337,13 +341,16 @@ static int verify_io_u_crc32c(struct verify_header *hdr, struct io_u *io_u,
 
 	dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", io_u, hdr->len);
 
-	c = crc32c(p, hdr->len - hdr_size(hdr));
+	if (hdr->verify_type == VERIFY_CRC32C_INTEL)
+		c = crc32c_intel(p, hdr->len - hdr_size(hdr));
+	else
+		c = crc32c(p, hdr->len - hdr_size(hdr));
 
 	if (c != vh->crc32) {
 		log_err("crc32c: verify failed at %llu/%u\n",
 				io_u->offset + header_num * hdr->len, hdr->len);
 		log_err("crc32c: wanted %x, got %x\n", vh->crc32, c);
-		return EIO;
+		return EILSEQ;
 	}
 
 	return 0;
@@ -369,7 +376,7 @@ static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u,
 				io_u->offset + header_num * hdr->len, hdr->len);
 		hexdump(vh->md5_digest, sizeof(vh->md5_digest));
 		hexdump(md5_ctx.hash, sizeof(hash));
-		return EIO;
+		return EILSEQ;
 	}
 
 	return 0;
@@ -402,7 +409,7 @@ int verify_io_u_pattern(unsigned long pattern, unsigned long pattern_size,
 			log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
 				buf[i], split_pattern[mod], bits);
 			log_err("fio: bad pattern block offset %u\n", i);
-			return EIO;
+			return EILSEQ;
 		}
 		mod++;
 		if (mod == pattern_size)
@@ -412,6 +419,26 @@ int verify_io_u_pattern(unsigned long pattern, unsigned long pattern_size,
 	return 0;
 }
 
+/*
+ * Push IO verification to a separate thread
+ */
+int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
+{
+	if (io_u->file)
+		put_file_log(td, io_u->file);
+
+	io_u->file = NULL;
+
+	pthread_mutex_lock(&td->io_u_lock);
+	flist_del(&io_u->list);
+	flist_add_tail(&io_u->list, &td->verify_list);
+	pthread_mutex_unlock(&td->io_u_lock);
+
+	pthread_cond_signal(&td->verify_cond);
+	io_u->flags |= IO_U_F_FREE_DEF;
+	return 0;
+}
+
 int verify_io_u(struct thread_data *td, struct io_u *io_u)
 {
 	struct verify_header *hdr;
@@ -440,7 +467,7 @@ int verify_io_u(struct thread_data *td, struct io_u *io_u)
 
 		if (hdr->fio_magic != FIO_HDR_MAGIC) {
 			log_err("Bad verify header %x\n", hdr->fio_magic);
-			return EIO;
+			return EILSEQ;
 		}
 
 		if (td->o.verify_pattern_bytes) {
@@ -451,6 +478,12 @@ int verify_io_u(struct thread_data *td, struct io_u *io_u)
 						  p + hdr_size,
 						  hdr_inc - hdr_size,
 						  hdr_size % 4);
+			/*
+			 * Also verify the meta data, if applicable
+			 */
+			if (hdr->verify_type == VERIFY_META)
+				ret |= verify_io_u_meta(hdr, td, io_u, hdr_num);
+
 			if (ret)
 				log_err("fio: verify failed at %llu/%u\n",
 					io_u->offset + hdr_num * hdr->len,
@@ -466,6 +499,7 @@ int verify_io_u(struct thread_data *td, struct io_u *io_u)
 			ret = verify_io_u_crc64(hdr, io_u, hdr_num);
 			break;
 		case VERIFY_CRC32C:
+		case VERIFY_CRC32C_INTEL:
 			ret = verify_io_u_crc32c(hdr, io_u, hdr_num);
 			break;
 		case VERIFY_CRC32:
@@ -557,7 +591,10 @@ static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
 {
 	struct vhdr_crc32 *vh = hdr_priv(hdr);
 
-	vh->crc32 = crc32c(p, len);
+	if (hdr->verify_type == VERIFY_CRC32C_INTEL)
+		vh->crc32 = crc32c_intel(p, len);
+	else
+		vh->crc32 = crc32c(p, len);
 }
 
 static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
@@ -618,6 +655,7 @@ void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
 			fill_crc64(hdr, data, data_len);
 			break;
 		case VERIFY_CRC32C:
+		case VERIFY_CRC32C_INTEL:
 			dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
 							io_u, hdr->len);
 			fill_crc32c(hdr, data, data_len);
@@ -687,7 +725,7 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u)
 		io_u->buflen = ipo->len;
 		io_u->file = ipo->file;
 
-		if ((io_u->file->flags & FIO_FILE_OPEN) == 0) {
+		if (!fio_file_open(io_u->file)) {
 			int r = td_io_open_file(td, io_u->file);
 
 			if (r) {
@@ -698,7 +736,7 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u)
 		}
 
 		get_file(ipo->file);
-		assert(io_u->file->flags & FIO_FILE_OPEN);
+		assert(fio_file_open(io_u->file));
 		io_u->ddir = DDIR_READ;
 		io_u->xfer_buf = io_u->buf;
 		io_u->xfer_buflen = io_u->buflen;
@@ -710,3 +748,122 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u)
 	dprint(FD_VERIFY, "get_next_verify: empty\n");
 	return 1;
 }
+
+static void *verify_async_thread(void *data)
+{
+	struct thread_data *td = data;
+	struct io_u *io_u;
+	int ret = 0;
+
+	if (td->o.verify_cpumask_set &&
+	    fio_setaffinity(td->pid, td->o.verify_cpumask)) {
+		log_err("fio: failed setting verify thread affinity\n");
+		goto done;
+	}
+
+	do {
+		FLIST_HEAD(list);
+
+		read_barrier();
+		if (td->verify_thread_exit)
+			break;
+
+		pthread_mutex_lock(&td->io_u_lock);
+
+		while (flist_empty(&td->verify_list) &&
+		       !td->verify_thread_exit) {
+			ret = pthread_cond_wait(&td->verify_cond,
+							&td->io_u_lock);
+			if (ret) {
+				pthread_mutex_unlock(&td->io_u_lock);
+				break;
+			}
+		}
+
+		flist_splice_init(&td->verify_list, &list);
+		pthread_mutex_unlock(&td->io_u_lock);
+
+		if (flist_empty(&list))
+			continue;
+
+		while (!flist_empty(&list)) {
+			io_u = flist_entry(list.next, struct io_u, list);
+			flist_del_init(&io_u->list);
+
+			ret = verify_io_u(td, io_u);
+			put_io_u(td, io_u);
+			if (!ret)
+				continue;
+			if (td->o.continue_on_error &&
+			    td_non_fatal_error(ret)) {
+				update_error_count(td, ret);
+				td_clear_error(td);
+				ret = 0;
+			}
+		}
+	} while (!ret);
+
+	if (ret) {
+		td_verror(td, ret, "async_verify");
+		td->terminate = 1;
+	}
+
+done:
+	pthread_mutex_lock(&td->io_u_lock);
+	td->nr_verify_threads--;
+	pthread_mutex_unlock(&td->io_u_lock);
+
+	pthread_cond_signal(&td->free_cond);
+	return NULL;
+}
+
+int verify_async_init(struct thread_data *td)
+{
+	int i, ret;
+
+	td->verify_thread_exit = 0;
+
+	td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
+	for (i = 0; i < td->o.verify_async; i++) {
+		ret = pthread_create(&td->verify_threads[i], NULL,
+					verify_async_thread, td);
+		if (ret) {
+			log_err("fio: async verify creation failed: %s\n",
+					strerror(ret));
+			break;
+		}
+		ret = pthread_detach(td->verify_threads[i]);
+		if (ret) {
+			log_err("fio: async verify thread detach failed: %s\n",
+					strerror(ret));
+			break;
+		}
+		td->nr_verify_threads++;
+	}
+
+	if (i != td->o.verify_async) {
+		log_err("fio: only %d verify threads started, exiting\n", i);
+		td->verify_thread_exit = 1;
+		write_barrier();
+		pthread_cond_broadcast(&td->verify_cond);
+		return 1;
+	}
+
+	return 0;
+}
+
+void verify_async_exit(struct thread_data *td)
+{
+	td->verify_thread_exit = 1;
+	write_barrier();
+	pthread_cond_broadcast(&td->verify_cond);
+
+	pthread_mutex_lock(&td->io_u_lock);
+
+	while (td->nr_verify_threads)
+		pthread_cond_wait(&td->free_cond, &td->io_u_lock);
+
+	pthread_mutex_unlock(&td->io_u_lock);
+	free(td->verify_threads);
+	td->verify_threads = NULL;
+}