[Qemu-devel] [kvm-unit-tests PATCH v7 08/11] arm/tlbflush-data: Add TLB flush during data writes test

[Alex Bennée]

This test is the cousin of the tlbflush-code test. Instead of flushing
running code it re-maps virtual addresses while a buffer is being filled
up. It then audits the results checking for writes that have ended up in
the wrong place.

While tlbflush-code exercises QEMU's translation invalidation logic this
test stresses the SoftMMU cputlb code and ensures it is semantically
correct.

The test optionally takes two parameters for debugging:

   cycles           - change the default number of test iterations
   page             - flush pages individually instead of all

Signed-off-by: Alex Bennée <address@hidden>
CC: Mark Rutland <address@hidden>
---
 arm/Makefile.common |   2 +
 arm/tlbflush-data.c | 401 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 arm/unittests.cfg   |  12 ++
 3 files changed, 415 insertions(+)
 create mode 100644 arm/tlbflush-data.c

diff --git a/arm/Makefile.common b/arm/Makefile.common
index de99a6e..528166d 100644
--- a/arm/Makefile.common
+++ b/arm/Makefile.common
@@ -14,6 +14,7 @@ tests-common += $(TEST_DIR)/spinlock-test.flat
 tests-common += $(TEST_DIR)/pci-test.flat
 tests-common += $(TEST_DIR)/gic.flat
 tests-common += $(TEST_DIR)/tlbflush-code.flat
+tests-common += $(TEST_DIR)/tlbflush-data.flat
 
 all: test_cases
 
@@ -83,3 +84,4 @@ test_cases: $(generated_files) $(tests-common) $(tests)
 
 $(TEST_DIR)/selftest.o $(cstart.o): $(asm-offsets)
 $(TEST_DIR)/tlbflush-code.elf: $(cstart.o) $(TEST_DIR)/tlbflush-code.o
+$(TEST_DIR)/tlbflush-data.elf: $(cstart.o) $(TEST_DIR)/tlbflush-data.o
diff --git a/arm/tlbflush-data.c b/arm/tlbflush-data.c
new file mode 100644
index 0000000..7920179
--- /dev/null
+++ b/arm/tlbflush-data.c
@@ -0,0 +1,401 @@
+/*
+ * TLB Flush Race Tests
+ *
+ * These tests are designed to test for incorrect TLB flush semantics
+ * under emulation. The initial CPU will set all the others working on
+ * a writing to a set of pages. It will then re-map one of the pages
+ * back and forth while recording the timestamps of when each page was
+ * active. The test fails if a write was detected on a page after the
+ * tlbflush switching to a new page should have completed.
+ *
+ * Copyright (C) 2016, Linaro, Alex Bennée <address@hidden>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.
+ */
+
+#include <libcflat.h>
+#include <asm/smp.h>
+#include <asm/cpumask.h>
+#include <asm/barrier.h>
+#include <asm/mmu.h>
+
+#define NR_TIMESTAMPS          ((PAGE_SIZE/sizeof(u64)) << 2)
+#define NR_AUDIT_RECORDS       16384
+#define NR_DYNAMIC_PAGES       3
+#define MAX_CPUS               8
+
+#define MIN(a, b)              ((a) < (b) ? (a) : (b))
+
+typedef struct {
+       u64             timestamps[NR_TIMESTAMPS];
+} write_buffer;
+
+typedef struct {
+       write_buffer    *newbuf;
+       u64             time_before_flush;
+       u64             time_after_flush;
+} audit_rec_t;
+
+typedef struct {
+       audit_rec_t     records[NR_AUDIT_RECORDS];
+} audit_buffer;
+
+typedef struct {
+       write_buffer    *stable_pages;
+       write_buffer    *dynamic_pages[NR_DYNAMIC_PAGES];
+       audit_buffer    *audit;
+       unsigned int    flush_count;
+} test_data_t;
+
+static test_data_t test_data[MAX_CPUS];
+
+static cpumask_t ready;
+static cpumask_t complete;
+
+static bool test_complete;
+static bool flush_verbose;
+static bool flush_by_page;
+static int test_cycles=3;
+static int secondary_cpus;
+
+static write_buffer * alloc_test_pages(void)
+{
+       write_buffer *pg;
+       pg = calloc(NR_TIMESTAMPS, sizeof(u64));
+       return pg;
+}
+
+static void setup_pages_for_cpu(int cpu)
+{
+       unsigned int i;
+
+       test_data[cpu].stable_pages = alloc_test_pages();
+
+       for (i=0; i<NR_DYNAMIC_PAGES; i++) {
+               test_data[cpu].dynamic_pages[i] = alloc_test_pages();
+       }
+
+       test_data[cpu].audit = calloc(NR_AUDIT_RECORDS, sizeof(audit_rec_t));
+}
+
+static audit_rec_t * get_audit_record(audit_buffer *buf, unsigned int record)
+{
+       return &buf->records[record];
+}
+
+/* Sync on a given cpumask */
+static void wait_on(int cpu, cpumask_t *mask)
+{
+       cpumask_set_cpu(cpu, mask);
+       while (!cpumask_full(mask))
+               cpu_relax();
+}
+
+static uint64_t sync_start(void)
+{
+       const uint64_t gate_mask = ~0x7ff;
+       uint64_t gate, now;
+       gate = get_cntvct() & gate_mask;
+       do {
+               now = get_cntvct();
+       } while ((now & gate_mask) == gate);
+
+       return now;
+}
+
+static void do_page_writes(void)
+{
+       unsigned int i, runs = 0;
+       int cpu = smp_processor_id();
+       write_buffer *stable_pages = test_data[cpu].stable_pages;
+       write_buffer *moving_page = test_data[cpu].dynamic_pages[0];
+
+       printf("CPU%d: ready %p/%p @ 0x%08" PRIx64"\n",
+               cpu, stable_pages, moving_page, get_cntvct());
+
+       while (!test_complete) {
+               u64 run_start, run_end;
+
+               smp_mb();
+               wait_on(cpu, &ready);
+               run_start = sync_start();
+
+               for (i = 0; i < NR_TIMESTAMPS; i++) {
+                       u64 ts = get_cntvct();
+                       moving_page->timestamps[i] = ts;
+                       stable_pages->timestamps[i] = ts;
+               }
+
+               run_end = get_cntvct();
+               printf("CPU%d: run %d 0x%" PRIx64 "->0x%" PRIx64 " (%" PRId64 " 
cycles)\n",
+                       cpu, runs++, run_start, run_end, run_end - run_start);
+
+               /* wait on completion - gets clear my main thread*/
+               wait_on(cpu, &complete);
+       }
+}
+
+
+/*
+ * This is the core of the test. Timestamps are taken either side of
+ * the updating of the page table and the flush instruction. By
+ * keeping track of when the page mapping is changed we can detect any
+ * writes that shouldn't have made it to the other pages.
+ *
+ * This isn't the recommended way to update the page table. ARM
+ * recommends break-before-make so accesses that are in flight can
+ * trigger faults that can be handled cleanly.
+ */
+
+/* This mimics  __flush_tlb_range from the kernel, doing a series of
+ * flush operations and then the dsb() to complete. */
+static void flush_pages(unsigned long start, unsigned long end)
+{
+       unsigned long addr;
+       start = start >> 12;
+       end = end >> 12;
+
+       dsb(ishst);
+       for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT -12)) {
+#if defined(__aarch64__)
+               asm("tlbi       vaae1is, %0" :: "r" (addr));
+#else
+               asm volatile("mcr p15, 0, %0, c8, c7, 3" :: "r" (addr));
+#endif
+       }
+       dsb(ish);
+}
+
+static void remap_one_page(test_data_t *data)
+{
+       u64 ts_before, ts_after;
+       int pg = (data->flush_count % (NR_DYNAMIC_PAGES + 1));
+       write_buffer *dynamic_pages_vaddr = data->dynamic_pages[0];
+       write_buffer *newbuf_paddr = data->dynamic_pages[pg];
+       write_buffer *end_page_paddr = newbuf_paddr+1;
+
+       ts_before = get_cntvct();
+       /* update the page table */
+       mmu_set_range_ptes(mmu_idmap,
+                       (unsigned long) dynamic_pages_vaddr,
+                       (unsigned long) newbuf_paddr,
+                       (unsigned long) end_page_paddr,
+                       __pgprot(PTE_WBWA));
+       /* until the flush + isb() writes may still go to old address */
+       if (flush_by_page) {
+               flush_pages((unsigned long)dynamic_pages_vaddr, (unsigned 
long)(dynamic_pages_vaddr+1));
+       } else {
+               flush_tlb_all();
+       }
+       ts_after = get_cntvct();
+
+       if (data->flush_count < NR_AUDIT_RECORDS) {
+               audit_rec_t *rec = get_audit_record(data->audit, 
data->flush_count);
+               rec->newbuf = newbuf_paddr;
+               rec->time_before_flush = ts_before;
+               rec->time_after_flush = ts_after;
+       }
+       data->flush_count++;
+}
+
+static int check_pages(int cpu, char *msg,
+               write_buffer *base_page, write_buffer *test_page,
+               audit_buffer *audit, unsigned int flushes)
+{
+       write_buffer *prev_page = base_page;
+       unsigned int empty = 0, write = 0, late = 0, weird = 0;
+       unsigned int ts_index = 0, audit_index;
+       u64 ts;
+
+       /* For each audit record */
+       for (audit_index = 0; audit_index < MIN(flushes, NR_AUDIT_RECORDS); 
audit_index++) {
+               audit_rec_t *rec = get_audit_record(audit, audit_index);
+
+               do {
+                       /* Work through timestamps until we overtake
+                        * this audit record */
+                       ts = test_page->timestamps[ts_index];
+
+                       if (ts == 0) {
+                               empty++;
+                       } else if (ts < rec->time_before_flush) {
+                               if (test_page == prev_page) {
+                                       write++;
+                               } else {
+                                       late++;
+                               }
+                       } else if (ts >= rec->time_before_flush
+                               && ts <= rec->time_after_flush) {
+                               if (test_page == prev_page
+                                       || test_page == rec->newbuf) {
+                                       write++;
+                               } else {
+                                       weird++;
+                               }
+                       } else if (ts > rec->time_after_flush) {
+                               if (test_page == rec->newbuf) {
+                                       write++;
+                               }
+                               /* It's possible the ts is way ahead
+                                * of the current record so we can't
+                                * call a non-match weird...
+                                *
+                                * Time to skip to next audit record
+                                */
+                               break;
+                       }
+
+                       ts = test_page->timestamps[ts_index++];
+               } while (ts <= rec->time_after_flush && ts_index < 
NR_TIMESTAMPS);
+
+
+               /* Next record */
+               prev_page = rec->newbuf;
+       } /* for each audit record */
+
+       if (flush_verbose) {
+               printf("CPU%d: %s %p => %p %u/%u/%u/%u (0/OK/L/?) = %u total\n",
+                       cpu, msg, test_page, base_page,
+                       empty, write, late, weird, empty+write+late+weird);
+       }
+
+       return weird;
+}
+
+static int audit_cpu_pages(int cpu, test_data_t *data)
+{
+       unsigned int pg, writes=0, ts_index = 0;
+       write_buffer *test_page;
+       int errors = 0;
+
+       /* first the stable page */
+       test_page = data->stable_pages;
+       do {
+               if (test_page->timestamps[ts_index++]) {
+                       writes++;
+               }
+       } while (ts_index < NR_TIMESTAMPS);
+
+       if (writes != ts_index) {
+               errors += 1;
+       }
+
+       if (flush_verbose) {
+               printf("CPU%d: stable page %p %u writes\n",
+                       cpu, test_page, writes);
+       }
+
+
+       /* Restore the mapping for dynamic page */
+       test_page = data->dynamic_pages[0];
+
+       mmu_set_range_ptes(mmu_idmap,
+                       (unsigned long) test_page,
+                       (unsigned long) test_page,
+                       (unsigned long) &test_page[1],
+                       __pgprot(PTE_WBWA));
+       flush_tlb_all();
+
+       for (pg=0; pg<NR_DYNAMIC_PAGES; pg++) {
+               errors += check_pages(cpu, "dynamic page", test_page,
+                               data->dynamic_pages[pg],
+                               data->audit, data->flush_count);
+       }
+
+       /* reset for next run */
+       memset(data->stable_pages, 0, sizeof(write_buffer));
+       for (pg=0; pg<NR_DYNAMIC_PAGES; pg++) {
+               memset(data->dynamic_pages[pg], 0, sizeof(write_buffer));
+       }
+       memset(data->audit, 0, sizeof(audit_buffer));
+       data->flush_count = 0;
+       smp_mb();
+
+       report("CPU%d: checked, errors: %d", errors == 0, cpu, errors);
+       return errors;
+}
+
+static void do_page_flushes(void)
+{
+       int i, cpu;
+
+       printf("CPU0: ready @ 0x%08" PRIx64"\n", get_cntvct());
+
+       for (i=0; i<test_cycles; i++) {
+               unsigned int flushes=0;
+               u64 run_start, run_end;
+               int cpus_finished;
+
+               cpumask_clear(&complete);
+               wait_on(0, &ready);
+               run_start = sync_start();
+
+               do {
+                       for_each_present_cpu(cpu) {
+                               if (cpu == 0)
+                                       continue;
+
+                               /* do remap & flush */
+                               remap_one_page(&test_data[cpu]);
+                               flushes++;
+                       }
+
+                       cpus_finished = cpumask_weight(&complete);
+               } while (cpus_finished < secondary_cpus);
+
+               run_end = get_cntvct();
+
+               printf("CPU0: run %d 0x%" PRIx64 "->0x%" PRIx64 " (%" PRId64 " 
cycles, %u flushes)\n",
+                       i, run_start, run_end, run_end - run_start, flushes);
+
+               /* Reset our ready mask for next cycle */
+               cpumask_clear_cpu(0, &ready);
+               smp_mb();
+               wait_on(0, &complete);
+
+               /* Check for discrepancies */
+               for_each_present_cpu(cpu) {
+                       if (cpu == 0)
+                               continue;
+                       audit_cpu_pages(cpu, &test_data[cpu]);
+               }
+       }
+
+       test_complete = true;
+       smp_mb();
+       cpumask_set_cpu(0, &ready);
+       cpumask_set_cpu(0, &complete);
+}
+
+int main(int argc, char **argv)
+{
+       int cpu, i;
+
+       for (i=0; i<argc; i++) {
+               char *arg = argv[i];
+               if (strcmp(arg, "verbose") == 0) {
+                       flush_verbose = true;
+               }
+               if (strcmp(arg, "page") == 0) {
+                       flush_by_page = true;
+               }
+               if (strstr(arg, "cycles=") != NULL) {
+                       char *p = strstr(arg, "=");
+                       test_cycles = atol(p+1);
+               }
+       }
+
+       for_each_present_cpu(cpu) {
+               if (cpu == 0)
+                       continue;
+
+               setup_pages_for_cpu(cpu);
+               smp_boot_secondary(cpu, do_page_writes);
+               secondary_cpus++;
+       }
+
+       /* CPU 0 does the flushes and checks the results */
+       do_page_flushes();
+
+       return report_summary();
+}
diff --git a/arm/unittests.cfg b/arm/unittests.cfg
index beaae84..7dc7799 100644
--- a/arm/unittests.cfg
+++ b/arm/unittests.cfg
@@ -96,3 +96,15 @@ file = tlbflush-code.flat
 smp = $(($MAX_SMP>4?4:$MAX_SMP))
 extra_params = -append 'page self'
 groups = tlbflush
+
+[tlbflush-data::all]
+file = tlbflush-data.flat
+smp = $(($MAX_SMP>4?4:$MAX_SMP))
+groups = tlbflush
+
+[tlbflush-data::page]
+file = tlbflush-data.flat
+smp = $(($MAX_SMP>4?4:$MAX_SMP))
+extra_params = -append "page"
+groups = tlbflush
+
-- 
2.10.1