summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/Kconfig4
-rw-r--r--lib/Kconfig.debug1
-rw-r--r--lib/Kconfig.kasan8
-rw-r--r--lib/cpumask.c102
-rw-r--r--lib/devres.c28
-rw-r--r--lib/find_last_bit.c41
-rw-r--r--lib/iommu-common.c192
-rw-r--r--lib/percpu_counter.c6
-rw-r--r--lib/raid6/algos.c41
-rw-r--r--lib/raid6/altivec.uc1
-rw-r--r--lib/raid6/avx2.c3
-rw-r--r--lib/raid6/int.uc41
-rw-r--r--lib/raid6/mmx.c2
-rw-r--r--lib/raid6/neon.c1
-rw-r--r--lib/raid6/sse1.c2
-rw-r--r--lib/raid6/sse2.c227
-rw-r--r--lib/raid6/test/test.c51
-rw-r--r--lib/raid6/tilegx.uc1
-rw-r--r--lib/rhashtable.c31
-rw-r--r--lib/string.c2
-rw-r--r--lib/strnlen_user.c12
-rw-r--r--lib/swiotlb.c5
-rw-r--r--lib/test-hexdump.c2
-rw-r--r--lib/test_bpf.c2664
-rw-r--r--lib/test_rhashtable.c215
25 files changed, 3344 insertions, 339 deletions
diff --git a/lib/Kconfig b/lib/Kconfig
index f5440221d929..601965a948e8 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -396,10 +396,6 @@ config CPUMASK_OFFSTACK
them on the stack. This is a bit more expensive, but avoids
stack overflow.
-config DISABLE_OBSOLETE_CPUMASK_FUNCTIONS
- bool "Disable obsolete cpumask functions" if DEBUG_PER_CPU_MAPS
- depends on BROKEN
-
config CPU_RMAP
bool
depends on SMP
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 17670573dda8..ba2b0c87e65b 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1281,6 +1281,7 @@ config RCU_TORTURE_TEST_SLOW_INIT_DELAY
int "How much to slow down RCU grace-period initialization"
range 0 5
default 3
+ depends on RCU_TORTURE_TEST_SLOW_INIT
help
This option specifies the number of jiffies to wait between
each rcu_node structure initialization.
diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan
index 4fecaedc80a2..777eda7d1ab4 100644
--- a/lib/Kconfig.kasan
+++ b/lib/Kconfig.kasan
@@ -10,8 +10,11 @@ config KASAN
help
Enables kernel address sanitizer - runtime memory debugger,
designed to find out-of-bounds accesses and use-after-free bugs.
- This is strictly debugging feature. It consumes about 1/8
- of available memory and brings about ~x3 performance slowdown.
+ This is strictly a debugging feature and it requires a gcc version
+ of 4.9.2 or later. Detection of out of bounds accesses to stack or
+ global variables requires gcc 5.0 or later.
+ This feature consumes about 1/8 of available memory and brings about
+ ~x3 performance slowdown.
For better error detection enable CONFIG_STACKTRACE,
and add slub_debug=U to boot cmdline.
@@ -40,6 +43,7 @@ config KASAN_INLINE
memory accesses. This is faster than outline (in some workloads
it gives about x2 boost over outline instrumentation), but
make kernel's .text size much bigger.
+ This requires a gcc version of 5.0 or later.
endchoice
diff --git a/lib/cpumask.c b/lib/cpumask.c
index 5ab1553fd076..5f627084f2e9 100644
--- a/lib/cpumask.c
+++ b/lib/cpumask.c
@@ -5,27 +5,6 @@
#include <linux/export.h>
#include <linux/bootmem.h>
-int __first_cpu(const cpumask_t *srcp)
-{
- return min_t(int, NR_CPUS, find_first_bit(srcp->bits, NR_CPUS));
-}
-EXPORT_SYMBOL(__first_cpu);
-
-int __next_cpu(int n, const cpumask_t *srcp)
-{
- return min_t(int, NR_CPUS, find_next_bit(srcp->bits, NR_CPUS, n+1));
-}
-EXPORT_SYMBOL(__next_cpu);
-
-#if NR_CPUS > 64
-int __next_cpu_nr(int n, const cpumask_t *srcp)
-{
- return min_t(int, nr_cpu_ids,
- find_next_bit(srcp->bits, nr_cpu_ids, n+1));
-}
-EXPORT_SYMBOL(__next_cpu_nr);
-#endif
-
/**
* cpumask_next_and - get the next cpu in *src1p & *src2p
* @n: the cpu prior to the place to search (ie. return will be > @n)
@@ -90,13 +69,6 @@ bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
dump_stack();
}
#endif
- /* FIXME: Bandaid to save us from old primitives which go to NR_CPUS. */
- if (*mask) {
- unsigned char *ptr = (unsigned char *)cpumask_bits(*mask);
- unsigned int tail;
- tail = BITS_TO_LONGS(NR_CPUS - nr_cpumask_bits) * sizeof(long);
- memset(ptr + cpumask_size() - tail, 0, tail);
- }
return *mask != NULL;
}
@@ -167,64 +139,42 @@ void __init free_bootmem_cpumask_var(cpumask_var_t mask)
#endif
/**
- * cpumask_set_cpu_local_first - set i'th cpu with local numa cpu's first
- *
+ * cpumask_local_spread - select the i'th cpu with local numa cpu's first
* @i: index number
- * @numa_node: local numa_node
- * @dstp: cpumask with the relevant cpu bit set according to the policy
+ * @node: local numa_node
*
- * This function sets the cpumask according to a numa aware policy.
- * cpumask could be used as an affinity hint for the IRQ related to a
- * queue. When the policy is to spread queues across cores - local cores
- * first.
+ * This function selects an online CPU according to a numa aware policy;
+ * local cpus are returned first, followed by non-local ones, then it
+ * wraps around.
*
- * Returns 0 on success, -ENOMEM for no memory, and -EAGAIN when failed to set
- * the cpu bit and need to re-call the function.
+ * It's not very efficient, but useful for setup.
*/
-int cpumask_set_cpu_local_first(int i, int numa_node, cpumask_t *dstp)
+unsigned int cpumask_local_spread(unsigned int i, int node)
{
- cpumask_var_t mask;
int cpu;
- int ret = 0;
-
- if (!zalloc_cpumask_var(&mask, GFP_KERNEL))
- return -ENOMEM;
+ /* Wrap: we always want a cpu. */
i %= num_online_cpus();
- if (numa_node == -1 || !cpumask_of_node(numa_node)) {
- /* Use all online cpu's for non numa aware system */
- cpumask_copy(mask, cpu_online_mask);
+ if (node == -1) {
+ for_each_cpu(cpu, cpu_online_mask)
+ if (i-- == 0)
+ return cpu;
} else {
- int n;
-
- cpumask_and(mask,
- cpumask_of_node(numa_node), cpu_online_mask);
-
- n = cpumask_weight(mask);
- if (i >= n) {
- i -= n;
-
- /* If index > number of local cpu's, mask out local
- * cpu's
- */
- cpumask_andnot(mask, cpu_online_mask, mask);
+ /* NUMA first. */
+ for_each_cpu_and(cpu, cpumask_of_node(node), cpu_online_mask)
+ if (i-- == 0)
+ return cpu;
+
+ for_each_cpu(cpu, cpu_online_mask) {
+ /* Skip NUMA nodes, done above. */
+ if (cpumask_test_cpu(cpu, cpumask_of_node(node)))
+ continue;
+
+ if (i-- == 0)
+ return cpu;
}
}
-
- for_each_cpu(cpu, mask) {
- if (--i < 0)
- goto out;
- }
-
- ret = -EAGAIN;
-
-out:
- free_cpumask_var(mask);
-
- if (!ret)
- cpumask_set_cpu(cpu, dstp);
-
- return ret;
+ BUG();
}
-EXPORT_SYMBOL(cpumask_set_cpu_local_first);
+EXPORT_SYMBOL(cpumask_local_spread);
diff --git a/lib/devres.c b/lib/devres.c
index 0f1dd2e9d2c1..fbe2aac522e6 100644
--- a/lib/devres.c
+++ b/lib/devres.c
@@ -72,6 +72,34 @@ void __iomem *devm_ioremap_nocache(struct device *dev, resource_size_t offset,
EXPORT_SYMBOL(devm_ioremap_nocache);
/**
+ * devm_ioremap_wc - Managed ioremap_wc()
+ * @dev: Generic device to remap IO address for
+ * @offset: BUS offset to map
+ * @size: Size of map
+ *
+ * Managed ioremap_wc(). Map is automatically unmapped on driver detach.
+ */
+void __iomem *devm_ioremap_wc(struct device *dev, resource_size_t offset,
+ resource_size_t size)
+{
+ void __iomem **ptr, *addr;
+
+ ptr = devres_alloc(devm_ioremap_release, sizeof(*ptr), GFP_KERNEL);
+ if (!ptr)
+ return NULL;
+
+ addr = ioremap_wc(offset, size);
+ if (addr) {
+ *ptr = addr;
+ devres_add(dev, ptr);
+ } else
+ devres_free(ptr);
+
+ return addr;
+}
+EXPORT_SYMBOL(devm_ioremap_wc);
+
+/**
* devm_iounmap - Managed iounmap()
* @dev: Generic device to unmap for
* @addr: Address to unmap
diff --git a/lib/find_last_bit.c b/lib/find_last_bit.c
deleted file mode 100644
index 3e3be40c6a6e..000000000000
--- a/lib/find_last_bit.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/* find_last_bit.c: fallback find next bit implementation
- *
- * Copyright (C) 2008 IBM Corporation
- * Written by Rusty Russell <rusty@rustcorp.com.au>
- * (Inspired by David Howell's find_next_bit implementation)
- *
- * Rewritten by Yury Norov <yury.norov@gmail.com> to decrease
- * size and improve performance, 2015.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/bitops.h>
-#include <linux/bitmap.h>
-#include <linux/export.h>
-#include <linux/kernel.h>
-
-#ifndef find_last_bit
-
-unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
-{
- if (size) {
- unsigned long val = BITMAP_LAST_WORD_MASK(size);
- unsigned long idx = (size-1) / BITS_PER_LONG;
-
- do {
- val &= addr[idx];
- if (val)
- return idx * BITS_PER_LONG + __fls(val);
-
- val = ~0ul;
- } while (idx--);
- }
- return size;
-}
-EXPORT_SYMBOL(find_last_bit);
-
-#endif
diff --git a/lib/iommu-common.c b/lib/iommu-common.c
index fac4f35250c9..df30632f0bef 100644
--- a/lib/iommu-common.c
+++ b/lib/iommu-common.c
@@ -9,37 +9,72 @@
#include <linux/iommu-helper.h>
#include <linux/iommu-common.h>
#include <linux/dma-mapping.h>
+#include <linux/hash.h>
#ifndef DMA_ERROR_CODE
#define DMA_ERROR_CODE (~(dma_addr_t)0x0)
#endif
-#define IOMMU_LARGE_ALLOC 15
+static unsigned long iommu_large_alloc = 15;
+
+static DEFINE_PER_CPU(unsigned int, iommu_hash_common);
+
+static inline bool need_flush(struct iommu_map_table *iommu)
+{
+ return (iommu->lazy_flush != NULL &&
+ (iommu->flags & IOMMU_NEED_FLUSH) != 0);
+}
+
+static inline void set_flush(struct iommu_map_table *iommu)
+{
+ iommu->flags |= IOMMU_NEED_FLUSH;
+}
+
+static inline void clear_flush(struct iommu_map_table *iommu)
+{
+ iommu->flags &= ~IOMMU_NEED_FLUSH;
+}
+
+static void setup_iommu_pool_hash(void)
+{
+ unsigned int i;
+ static bool do_once;
+
+ if (do_once)
+ return;
+ do_once = true;
+ for_each_possible_cpu(i)
+ per_cpu(iommu_hash_common, i) = hash_32(i, IOMMU_POOL_HASHBITS);
+}
/*
- * Initialize iommu_pool entries for the iommu_table. `num_entries'
+ * Initialize iommu_pool entries for the iommu_map_table. `num_entries'
* is the number of table entries. If `large_pool' is set to true,
* the top 1/4 of the table will be set aside for pool allocations
- * of more than IOMMU_LARGE_ALLOC pages.
+ * of more than iommu_large_alloc pages.
*/
-extern void iommu_tbl_pool_init(struct iommu_table *iommu,
- unsigned long num_entries,
- u32 page_table_shift,
- const struct iommu_tbl_ops *iommu_tbl_ops,
- bool large_pool, u32 npools)
+void iommu_tbl_pool_init(struct iommu_map_table *iommu,
+ unsigned long num_entries,
+ u32 table_shift,
+ void (*lazy_flush)(struct iommu_map_table *),
+ bool large_pool, u32 npools,
+ bool skip_span_boundary_check)
{
unsigned int start, i;
struct iommu_pool *p = &(iommu->large_pool);
+ setup_iommu_pool_hash();
if (npools == 0)
iommu->nr_pools = IOMMU_NR_POOLS;
else
iommu->nr_pools = npools;
BUG_ON(npools > IOMMU_NR_POOLS);
- iommu->page_table_shift = page_table_shift;
- iommu->iommu_tbl_ops = iommu_tbl_ops;
+ iommu->table_shift = table_shift;
+ iommu->lazy_flush = lazy_flush;
start = 0;
+ if (skip_span_boundary_check)
+ iommu->flags |= IOMMU_NO_SPAN_BOUND;
if (large_pool)
iommu->flags |= IOMMU_HAS_LARGE_POOL;
@@ -48,11 +83,11 @@ extern void iommu_tbl_pool_init(struct iommu_table *iommu,
else
iommu->poolsize = (num_entries * 3 / 4)/iommu->nr_pools;
for (i = 0; i < iommu->nr_pools; i++) {
- spin_lock_init(&(iommu->arena_pool[i].lock));
- iommu->arena_pool[i].start = start;
- iommu->arena_pool[i].hint = start;
+ spin_lock_init(&(iommu->pools[i].lock));
+ iommu->pools[i].start = start;
+ iommu->pools[i].hint = start;
start += iommu->poolsize; /* start for next pool */
- iommu->arena_pool[i].end = start - 1;
+ iommu->pools[i].end = start - 1;
}
if (!large_pool)
return;
@@ -65,121 +100,136 @@ extern void iommu_tbl_pool_init(struct iommu_table *iommu,
EXPORT_SYMBOL(iommu_tbl_pool_init);
unsigned long iommu_tbl_range_alloc(struct device *dev,
- struct iommu_table *iommu,
+ struct iommu_map_table *iommu,
unsigned long npages,
unsigned long *handle,
- unsigned int pool_hash)
+ unsigned long mask,
+ unsigned int align_order)
{
+ unsigned int pool_hash = __this_cpu_read(iommu_hash_common);
unsigned long n, end, start, limit, boundary_size;
- struct iommu_pool *arena;
+ struct iommu_pool *pool;
int pass = 0;
unsigned int pool_nr;
unsigned int npools = iommu->nr_pools;
unsigned long flags;
bool large_pool = ((iommu->flags & IOMMU_HAS_LARGE_POOL) != 0);
- bool largealloc = (large_pool && npages > IOMMU_LARGE_ALLOC);
+ bool largealloc = (large_pool && npages > iommu_large_alloc);
unsigned long shift;
+ unsigned long align_mask = 0;
+
+ if (align_order > 0)
+ align_mask = 0xffffffffffffffffl >> (64 - align_order);
/* Sanity check */
if (unlikely(npages == 0)) {
- printk_ratelimited("npages == 0\n");
+ WARN_ON_ONCE(1);
return DMA_ERROR_CODE;
}
if (largealloc) {
- arena = &(iommu->large_pool);
- spin_lock_irqsave(&arena->lock, flags);
+ pool = &(iommu->large_pool);
pool_nr = 0; /* to keep compiler happy */
} else {
/* pick out pool_nr */
pool_nr = pool_hash & (npools - 1);
- arena = &(iommu->arena_pool[pool_nr]);
-
- /* find first available unlocked pool */
- while (!spin_trylock_irqsave(&(arena->lock), flags)) {
- pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1);
- arena = &(iommu->arena_pool[pool_nr]);
- }
+ pool = &(iommu->pools[pool_nr]);
}
+ spin_lock_irqsave(&pool->lock, flags);
again:
if (pass == 0 && handle && *handle &&
- (*handle >= arena->start) && (*handle < arena->end))
+ (*handle >= pool->start) && (*handle < pool->end))
start = *handle;
else
- start = arena->hint;
+ start = pool->hint;
- limit = arena->end;
+ limit = pool->end;
/* The case below can happen if we have a small segment appended
* to a large, or when the previous alloc was at the very end of
- * the available space. If so, go back to the beginning and flush.
+ * the available space. If so, go back to the beginning. If a
+ * flush is needed, it will get done based on the return value
+ * from iommu_area_alloc() below.
*/
- if (start >= limit) {
- start = arena->start;
- if (iommu->iommu_tbl_ops->reset != NULL)
- iommu->iommu_tbl_ops->reset(iommu);
+ if (start >= limit)
+ start = pool->start;
+ shift = iommu->table_map_base >> iommu->table_shift;
+ if (limit + shift > mask) {
+ limit = mask - shift + 1;
+ /* If we're constrained on address range, first try
+ * at the masked hint to avoid O(n) search complexity,
+ * but on second pass, start at 0 in pool 0.
+ */
+ if ((start & mask) >= limit || pass > 0) {
+ spin_unlock(&(pool->lock));
+ pool = &(iommu->pools[0]);
+ spin_lock(&(pool->lock));
+ start = pool->start;
+ } else {
+ start &= mask;
+ }
}
if (dev)
boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
- 1 << iommu->page_table_shift);
+ 1 << iommu->table_shift);
else
- boundary_size = ALIGN(1ULL << 32, 1 << iommu->page_table_shift);
+ boundary_size = ALIGN(1ULL << 32, 1 << iommu->table_shift);
- shift = iommu->page_table_map_base >> iommu->page_table_shift;
- boundary_size = boundary_size >> iommu->page_table_shift;
+ boundary_size = boundary_size >> iommu->table_shift;
/*
- * if the iommu has a non-trivial cookie <-> index mapping, we set
+ * if the skip_span_boundary_check had been set during init, we set
* things up so that iommu_is_span_boundary() merely checks if the
* (index + npages) < num_tsb_entries
*/
- if (iommu->iommu_tbl_ops->cookie_to_index != NULL) {
+ if ((iommu->flags & IOMMU_NO_SPAN_BOUND) != 0) {
shift = 0;
boundary_size = iommu->poolsize * iommu->nr_pools;
}
n = iommu_area_alloc(iommu->map, limit, start, npages, shift,
- boundary_size, 0);
+ boundary_size, align_mask);
if (n == -1) {
if (likely(pass == 0)) {
/* First failure, rescan from the beginning. */
- arena->hint = arena->start;
- if (iommu->iommu_tbl_ops->reset != NULL)
- iommu->iommu_tbl_ops->reset(iommu);
+ pool->hint = pool->start;
+ set_flush(iommu);
pass++;
goto again;
} else if (!largealloc && pass <= iommu->nr_pools) {
- spin_unlock(&(arena->lock));
+ spin_unlock(&(pool->lock));
pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1);
- arena = &(iommu->arena_pool[pool_nr]);
- while (!spin_trylock(&(arena->lock))) {
- pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1);
- arena = &(iommu->arena_pool[pool_nr]);
- }
- arena->hint = arena->start;
+ pool = &(iommu->pools[pool_nr]);
+ spin_lock(&(pool->lock));
+ pool->hint = pool->start;
+ set_flush(iommu);
pass++;
goto again;
} else {
/* give up */
- spin_unlock_irqrestore(&(arena->lock), flags);
- return DMA_ERROR_CODE;
+ n = DMA_ERROR_CODE;
+ goto bail;
}
}
+ if (n < pool->hint || need_flush(iommu)) {
+ clear_flush(iommu);
+ iommu->lazy_flush(iommu);
+ }
end = n + npages;
-
- arena->hint = end;
+ pool->hint = end;
/* Update handle for SG allocations */
if (handle)
*handle = end;
- spin_unlock_irqrestore(&(arena->lock), flags);
+bail:
+ spin_unlock_irqrestore(&(pool->lock), flags);
return n;
}
EXPORT_SYMBOL(iommu_tbl_range_alloc);
-static struct iommu_pool *get_pool(struct iommu_table *tbl,
+static struct iommu_pool *get_pool(struct iommu_map_table *tbl,
unsigned long entry)
{
struct iommu_pool *p;
@@ -193,31 +243,27 @@ static struct iommu_pool *get_pool(struct iommu_table *tbl,
unsigned int pool_nr = entry / tbl->poolsize;
BUG_ON(pool_nr >= tbl->nr_pools);
- p = &tbl->arena_pool[pool_nr];
+ p = &tbl->pools[pool_nr];
}
return p;
}
-void iommu_tbl_range_free(struct iommu_table *iommu, u64 dma_addr,
- unsigned long npages, bool do_demap, void *demap_arg)
+/* Caller supplies the index of the entry into the iommu map table
+ * itself when the mapping from dma_addr to the entry is not the
+ * default addr->entry mapping below.
+ */
+void iommu_tbl_range_free(struct iommu_map_table *iommu, u64 dma_addr,
+ unsigned long npages, unsigned long entry)
{
- unsigned long entry;
struct iommu_pool *pool;
unsigned long flags;
- unsigned long shift = iommu->page_table_shift;
+ unsigned long shift = iommu->table_shift;
- if (iommu->iommu_tbl_ops->cookie_to_index != NULL) {
- entry = (*iommu->iommu_tbl_ops->cookie_to_index)(dma_addr,
- demap_arg);
- } else {
- entry = (dma_addr - iommu->page_table_map_base) >> shift;
- }
+ if (entry == DMA_ERROR_CODE) /* use default addr->entry mapping */
+ entry = (dma_addr - iommu->table_map_base) >> shift;
pool = get_pool(iommu, entry);
spin_lock_irqsave(&(pool->lock), flags);
- if (do_demap && iommu->iommu_tbl_ops->demap != NULL)
- (*iommu->iommu_tbl_ops->demap)(demap_arg, entry, npages);
-
bitmap_clear(iommu->map, entry, npages);
spin_unlock_irqrestore(&(pool->lock), flags);
}
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index 48144cdae819..f051d69f0910 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -197,13 +197,13 @@ static int percpu_counter_hotcpu_callback(struct notifier_block *nb,
* Compare counter against given value.
* Return 1 if greater, 0 if equal and -1 if less
*/
-int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
+int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)
{
s64 count;
count = percpu_counter_read(fbc);
/* Check to see if rough count will be sufficient for comparison */
- if (abs(count - rhs) > (percpu_counter_batch*num_online_cpus())) {
+ if (abs(count - rhs) > (batch * num_online_cpus())) {
if (count > rhs)
return 1;
else
@@ -218,7 +218,7 @@ int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
else
return 0;
}
-EXPORT_SYMBOL(percpu_counter_compare);
+EXPORT_SYMBOL(__percpu_counter_compare);
static int __init percpu_counter_startup(void)
{
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index dbef2314901e..975c6e0434bd 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -131,11 +131,12 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void)
static inline const struct raid6_calls *raid6_choose_gen(
void *(*const dptrs)[(65536/PAGE_SIZE)+2], const int disks)
{
- unsigned long perf, bestperf, j0, j1;
+ unsigned long perf, bestgenperf, bestxorperf, j0, j1;
+ int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */
const struct raid6_calls *const *algo;
const struct raid6_calls *best;
- for (bestperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) {
+ for (bestgenperf = 0, bestxorperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) {
if (!best || (*algo)->prefer >= best->prefer) {
if ((*algo)->valid && !(*algo)->valid())
continue;
@@ -153,19 +154,45 @@ static inline const struct raid6_calls *raid6_choose_gen(
}
preempt_enable();
- if (perf > bestperf) {
- bestperf = perf;
+ if (perf > bestgenperf) {
+ bestgenperf = perf;
best = *algo;
}
- pr_info("raid6: %-8s %5ld MB/s\n", (*algo)->name,
+ pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name,
(perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2));
+
+ if (!(*algo)->xor_syndrome)
+ continue;
+
+ perf = 0;
+
+ preempt_disable();
+ j0 = jiffies;
+ while ((j1 = jiffies) == j0)
+ cpu_relax();
+ while (time_before(jiffies,
+ j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
+ (*algo)->xor_syndrome(disks, start, stop,
+ PAGE_SIZE, *dptrs);
+ perf++;
+ }
+ preempt_enable();
+
+ if (best == *algo)
+ bestxorperf = perf;
+
+ pr_info("raid6: %-8s xor() %5ld MB/s\n", (*algo)->name,
+ (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1));
}
}
if (best) {
- pr_info("raid6: using algorithm %s (%ld MB/s)\n",
+ pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
best->name,
- (bestperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2));
+ (bestgenperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2));
+ if (best->xor_syndrome)
+ pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
+ (bestxorperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1));
raid6_call = *best;
} else
pr_err("raid6: Yikes! No algorithm found!\n");
diff --git a/lib/raid6/altivec.uc b/lib/raid6/altivec.uc
index 7cc12b532e95..bec27fce7501 100644
--- a/lib/raid6/altivec.uc
+++ b/lib/raid6/altivec.uc
@@ -119,6 +119,7 @@ int raid6_have_altivec(void)
const struct raid6_calls raid6_altivec$# = {
raid6_altivec$#_gen_syndrome,
+ NULL, /* XOR not yet implemented */
raid6_have_altivec,
"altivecx$#",
0
diff --git a/lib/raid6/avx2.c b/lib/raid6/avx2.c
index bc3b1dd436eb..76734004358d 100644
--- a/lib/raid6/avx2.c
+++ b/lib/raid6/avx2.c
@@ -89,6 +89,7 @@ static void raid6_avx21_gen_syndrome(int disks, size_t bytes, void **ptrs)
const struct raid6_calls raid6_avx2x1 = {
raid6_avx21_gen_syndrome,
+ NULL, /* XOR not yet implemented */
raid6_have_avx2,
"avx2x1",
1 /* Has cache hints */
@@ -150,6 +151,7 @@ static void raid6_avx22_gen_syndrome(int disks, size_t bytes, void **ptrs)
const struct raid6_calls raid6_avx2x2 = {
raid6_avx22_gen_syndrome,
+ NULL, /* XOR not yet implemented */
raid6_have_avx2,
"avx2x2",
1 /* Has cache hints */
@@ -242,6 +244,7 @@ static void raid6_avx24_gen_syndrome(int disks, size_t bytes, void **ptrs)
const struct raid6_calls raid6_avx2x4 = {
raid6_avx24_gen_syndrome,
+ NULL, /* XOR not yet implemented */
raid6_have_avx2,
"avx2x4",
1 /* Has cache hints */
diff --git a/lib/raid6/int.uc b/lib/raid6/int.uc
index 5b50f8dfc5d2..558aeac9342a 100644
--- a/lib/raid6/int.uc
+++ b/lib/raid6/int.uc
@@ -107,9 +107,48 @@ static void raid6_int$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
}
}
+static void raid6_int$#_xor_syndrome(int disks, int start, int stop,
+ size_t bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ int d, z, z0;
+
+ unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
+
+ z0 = stop; /* P/Q right side optimization */
+ p = dptr[disks-2]; /* XOR parity */
+ q = dptr[disks-1]; /* RS syndrome */
+
+ for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
+ /* P/Q data pages */
+ wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE];
+ for ( z = z0-1 ; z >= start ; z-- ) {
+ wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+ wp$$ ^= wd$$;
+ w2$$ = MASK(wq$$);
+ w1$$ = SHLBYTE(wq$$);
+ w2$$ &= NBYTES(0x1d);
+ w1$$ ^= w2$$;
+ wq$$ = w1$$ ^ wd$$;
+ }
+ /* P/Q left side optimization */
+ for ( z = start-1 ; z >= 0 ; z-- ) {
+ w2$$ = MASK(wq$$);
+ w1$$ = SHLBYTE(wq$$);
+ w2$$ &= NBYTES(0x1d);
+ wq$$ = w1$$ ^ w2$$;
+ }
+ *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
+ *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
+ }
+
+}
+
const struct raid6_calls raid6_intx$# = {
raid6_int$#_gen_syndrome,
- NULL, /* always valid */
+ raid6_int$#_xor_syndrome,
+ NULL, /* always valid */
"int" NSTRING "x$#",
0
};
diff --git a/lib/raid6/mmx.c b/lib/raid6/mmx.c
index 590c71c9e200..b3b0e1fcd3af 100644
--- a/lib/raid6/mmx.c
+++ b/lib/raid6/mmx.c
@@ -76,6 +76,7 @@ static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
const struct raid6_calls raid6_mmxx1 = {
raid6_mmx1_gen_syndrome,
+ NULL, /* XOR not yet implemented */
raid6_have_mmx,
"mmxx1",
0
@@ -134,6 +135,7 @@ static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
const struct raid6_calls raid6_mmxx2 = {
raid6_mmx2_gen_syndrome,
+ NULL, /* XOR not yet implemented */
raid6_have_mmx,
"mmxx2",
0
diff --git a/lib/raid6/neon.c b/lib/raid6/neon.c
index 36ad4705df1a..d9ad6ee284f4 100644
--- a/lib/raid6/neon.c
+++ b/lib/raid6/neon.c
@@ -42,6 +42,7 @@
} \
struct raid6_calls const raid6_neonx ## _n = { \
raid6_neon ## _n ## _gen_syndrome, \
+ NULL, /* XOR not yet implemented */ \
raid6_have_neon, \
"neonx" #_n, \
0 \
diff --git a/lib/raid6/sse1.c b/lib/raid6/sse1.c
index f76297139445..9025b8ca9aa3 100644
--- a/lib/raid6/sse1.c
+++ b/lib/raid6/sse1.c
@@ -92,6 +92,7 @@ static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs)
const struct raid6_calls raid6_sse1x1 = {
raid6_sse11_gen_syndrome,
+ NULL, /* XOR not yet implemented */
raid6_have_sse1_or_mmxext,
"sse1x1",
1 /* Has cache hints */
@@ -154,6 +155,7 @@ static void raid6_sse12_gen_syndrome(int disks, size_t bytes, void **ptrs)
const struct raid6_calls raid6_sse1x2 = {
raid6_sse12_gen_syndrome,
+ NULL, /* XOR not yet implemented */
raid6_have_sse1_or_mmxext,
"sse1x2",
1 /* Has cache hints */
diff --git a/lib/raid6/sse2.c b/lib/raid6/sse2.c
index 85b82c85f28e..1d2276b007ee 100644
--- a/lib/raid6/sse2.c
+++ b/lib/raid6/sse2.c
@@ -88,8 +88,58 @@ static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs)
kernel_fpu_end();
}
+
+static void raid6_sse21_xor_syndrome(int disks, int start, int stop,
+ size_t bytes, void **ptrs)
+ {
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ int d, z, z0;
+
+ z0 = stop; /* P/Q right side optimization */
+ p = dptr[disks-2]; /* XOR parity */
+ q = dptr[disks-1]; /* RS syndrome */
+
+ kernel_fpu_begin();
+
+ asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
+
+ for ( d = 0 ; d < bytes ; d += 16 ) {
+ asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
+ asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
+ asm volatile("pxor %xmm4,%xmm2");
+ /* P/Q data pages */
+ for ( z = z0-1 ; z >= start ; z-- ) {
+ asm volatile("pxor %xmm5,%xmm5");
+ asm volatile("pcmpgtb %xmm4,%xmm5");
+ asm volatile("paddb %xmm4,%xmm4");
+ asm volatile("pand %xmm0,%xmm5");
+ asm volatile("pxor %xmm5,%xmm4");
+ asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
+ asm volatile("pxor %xmm5,%xmm2");
+ asm volatile("pxor %xmm5,%xmm4");
+ }
+ /* P/Q left side optimization */
+ for ( z = start-1 ; z >= 0 ; z-- ) {
+ asm volatile("pxor %xmm5,%xmm5");
+ asm volatile("pcmpgtb %xmm4,%xmm5");
+ asm volatile("paddb %xmm4,%xmm4");
+ asm volatile("pand %xmm0,%xmm5");
+ asm volatile("pxor %xmm5,%xmm4");
+ }
+ asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
+ /* Don't use movntdq for r/w memory area < cache line */
+ asm volatile("movdqa %%xmm4,%0" : "=m" (q[d]));
+ asm volatile("movdqa %%xmm2,%0" : "=m" (p[d]));
+ }
+
+ asm volatile("sfence" : : : "memory");
+ kernel_fpu_end();
+}
+
const struct raid6_calls raid6_sse2x1 = {
raid6_sse21_gen_syndrome,
+ raid6_sse21_xor_syndrome,
raid6_have_sse2,
"sse2x1",
1 /* Has cache hints */
@@ -150,8 +200,76 @@ static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs)
kernel_fpu_end();
}
+ static void raid6_sse22_xor_syndrome(int disks, int start, int stop,
+ size_t bytes, void **ptrs)
+ {
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ int d, z, z0;
+
+ z0 = stop; /* P/Q right side optimization */
+ p = dptr[disks-2]; /* XOR parity */
+ q = dptr[disks-1]; /* RS syndrome */
+
+ kernel_fpu_begin();
+
+ asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
+
+ for ( d = 0 ; d < bytes ; d += 32 ) {
+ asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
+ asm volatile("movdqa %0,%%xmm6" :: "m" (dptr[z0][d+16]));
+ asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
+ asm volatile("movdqa %0,%%xmm3" : : "m" (p[d+16]));
+ asm volatile("pxor %xmm4,%xmm2");
+ asm volatile("pxor %xmm6,%xmm3");
+ /* P/Q data pages */
+ for ( z = z0-1 ; z >= start ; z-- ) {
+ asm volatile("pxor %xmm5,%xmm5");
+ asm volatile("pxor %xmm7,%xmm7");
+ asm volatile("pcmpgtb %xmm4,%xmm5");
+ asm volatile("pcmpgtb %xmm6,%xmm7");
+ asm volatile("paddb %xmm4,%xmm4");
+ asm volatile("paddb %xmm6,%xmm6");
+ asm volatile("pand %xmm0,%xmm5");
+ asm volatile("pand %xmm0,%xmm7");
+ asm volatile("pxor %xmm5,%xmm4");
+ asm volatile("pxor %xmm7,%xmm6");
+ asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
+ asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
+ asm volatile("pxor %xmm5,%xmm2");
+ asm volatile("pxor %xmm7,%xmm3");
+ asm volatile("pxor %xmm5,%xmm4");
+ asm volatile("pxor %xmm7,%xmm6");
+ }
+ /* P/Q left side optimization */
+ for ( z = start-1 ; z >= 0 ; z-- ) {
+ asm volatile("pxor %xmm5,%xmm5");
+ asm volatile("pxor %xmm7,%xmm7");
+ asm volatile("pcmpgtb %xmm4,%xmm5");
+ asm volatile("pcmpgtb %xmm6,%xmm7");
+ asm volatile("paddb %xmm4,%xmm4");
+ asm volatile("paddb %xmm6,%xmm6");
+ asm volatile("pand %xmm0,%xmm5");
+ asm volatile("pand %xmm0,%xmm7");
+ asm volatile("pxor %xmm5,%xmm4");
+ asm volatile("pxor %xmm7,%xmm6");
+ }
+ asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
+ asm volatile("pxor %0,%%xmm6" : : "m" (q[d+16]));
+ /* Don't use movntdq for r/w memory area < cache line */
+ asm volatile("movdqa %%xmm4,%0" : "=m" (q[d]));
+ asm volatile("movdqa %%xmm6,%0" : "=m" (q[d+16]));
+ asm volatile("movdqa %%xmm2,%0" : "=m" (p[d]));
+ asm volatile("movdqa %%xmm3,%0" : "=m" (p[d+16]));
+ }
+
+ asm volatile("sfence" : : : "memory");
+ kernel_fpu_end();
+ }
+
const struct raid6_calls raid6_sse2x2 = {
raid6_sse22_gen_syndrome,
+ raid6_sse22_xor_syndrome,
raid6_have_sse2,
"sse2x2",
1 /* Has cache hints */
@@ -248,8 +366,117 @@ static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs)
kernel_fpu_end();
}
+ static void raid6_sse24_xor_syndrome(int disks, int start, int stop,
+ size_t bytes, void **ptrs)
+ {
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ int d, z, z0;
+
+ z0 = stop; /* P/Q right side optimization */
+ p = dptr[disks-2]; /* XOR parity */
+ q = dptr[disks-1]; /* RS syndrome */
+
+ kernel_fpu_begin();
+
+ asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
+
+ for ( d = 0 ; d < bytes ; d += 64 ) {
+ asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
+ asm volatile("movdqa %0,%%xmm6" :: "m" (dptr[z0][d+16]));
+ asm volatile("movdqa %0,%%xmm12" :: "m" (dptr[z0][d+32]));
+ asm volatile("movdqa %0,%%xmm14" :: "m" (dptr[z0][d+48]));
+ asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
+ asm volatile("movdqa %0,%%xmm3" : : "m" (p[d+16]));
+ asm volatile("movdqa %0,%%xmm10" : : "m" (p[d+32]));
+ asm volatile("movdqa %0,%%xmm11" : : "m" (p[d+48]));
+ asm volatile("pxor %xmm4,%xmm2");
+ asm volatile("pxor %xmm6,%xmm3");
+ asm volatile("pxor %xmm12,%xmm10");
+ asm volatile("pxor %xmm14,%xmm11");
+ /* P/Q data pages */
+ for ( z = z0-1 ; z >= start ; z-- ) {
+ asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
+ asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
+ asm volatile("pxor %xmm5,%xmm5");
+ asm volatile("pxor %xmm7,%xmm7");
+ asm volatile("pxor %xmm13,%xmm13");
+ asm volatile("pxor %xmm15,%xmm15");
+ asm volatile("pcmpgtb %xmm4,%xmm5");
+ asm volatile("pcmpgtb %xmm6,%xmm7");
+ asm volatile("pcmpgtb %xmm12,%xmm13");
+ asm volatile("pcmpgtb %xmm14,%xmm15");
+ asm volatile("paddb %xmm4,%xmm4");
+ asm volatile("paddb %xmm6,%xmm6");
+ asm volatile("paddb %xmm12,%xmm12");
+ asm volatile("paddb %xmm14,%xmm14");
+ asm volatile("pand %xmm0,%xmm5");
+ asm volatile("pand %xmm0,%xmm7");
+ asm volatile("pand %xmm0,%xmm13");
+ asm volatile("pand %xmm0,%xmm15");
+ asm volatile("pxor %xmm5,%xmm4");
+ asm volatile("pxor %xmm7,%xmm6");
+ asm volatile("pxor %xmm13,%xmm12");
+ asm volatile("pxor %xmm15,%xmm14");
+ asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
+ asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
+ asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
+ asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
+ asm volatile("pxor %xmm5,%xmm2");
+ asm volatile("pxor %xmm7,%xmm3");
+ asm volatile("pxor %xmm13,%xmm10");
+ asm volatile("pxor %xmm15,%xmm11");
+ asm volatile("pxor %xmm5,%xmm4");
+ asm volatile("pxor %xmm7,%xmm6");
+ asm volatile("pxor %xmm13,%xmm12");
+ asm volatile("pxor %xmm15,%xmm14");
+ }
+ asm volatile("prefetchnta %0" :: "m" (q[d]));
+ asm volatile("prefetchnta %0" :: "m" (q[d+32]));
+ /* P/Q left side optimization */
+ for ( z = start-1 ; z >= 0 ; z-- ) {
+ asm volatile("pxor %xmm5,%xmm5");
+ asm volatile("pxor %xmm7,%xmm7");
+ asm volatile("pxor %xmm13,%xmm13");
+ asm volatile("pxor %xmm15,%xmm15");
+ asm volatile("pcmpgtb %xmm4,%xmm5");
+ asm volatile("pcmpgtb %xmm6,%xmm7");
+ asm volatile("pcmpgtb %xmm12,%xmm13");
+ asm volatile("pcmpgtb %xmm14,%xmm15");
+ asm volatile("paddb %xmm4,%xmm4");
+ asm volatile("paddb %xmm6,%xmm6");
+ asm volatile("paddb %xmm12,%xmm12");
+ asm volatile("paddb %xmm14,%xmm14");
+ asm volatile("pand %xmm0,%xmm5");
+ asm volatile("pand %xmm0,%xmm7");
+ asm volatile("pand %xmm0,%xmm13");
+ asm volatile("pand %xmm0,%xmm15");
+ asm volatile("pxor %xmm5,%xmm4");
+ asm volatile("pxor %xmm7,%xmm6");
+ asm volatile("pxor %xmm13,%xmm12");
+ asm volatile("pxor %xmm15,%xmm14");
+ }
+ asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
+ asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
+ asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
+ asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
+ asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
+ asm volatile("pxor %0,%%xmm6" : : "m" (q[d+16]));
+ asm volatile("pxor %0,%%xmm12" : : "m" (q[d+32]));
+ asm volatile("pxor %0,%%xmm14" : : "m" (q[d+48]));
+ asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
+ asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
+ asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
+ asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
+ }
+ asm volatile("sfence" : : : "memory");
+ kernel_fpu_end();
+ }
+
+
const struct raid6_calls raid6_sse2x4 = {
raid6_sse24_gen_syndrome,
+ raid6_sse24_xor_syndrome,
raid6_have_sse2,
"sse2x4",
1 /* Has cache hints */
diff --git a/lib/raid6/test/test.c b/lib/raid6/test/test.c
index 5a485b7a7d3c..3bebbabdb510 100644
--- a/lib/raid6/test/test.c
+++ b/lib/raid6/test/test.c
@@ -28,11 +28,11 @@ char *dataptrs[NDISKS];
char data[NDISKS][PAGE_SIZE];
char recovi[PAGE_SIZE], recovj[PAGE_SIZE];
-static void makedata(void)
+static void makedata(int start, int stop)
{
int i, j;
- for (i = 0; i < NDISKS; i++) {
+ for (i = start; i <= stop; i++) {
for (j = 0; j < PAGE_SIZE; j++)
data[i][j] = rand();
@@ -91,34 +91,55 @@ int main(int argc, char *argv[])
{
const struct raid6_calls *const *algo;
const struct raid6_recov_calls *const *ra;
- int i, j;
+ int i, j, p1, p2;
int err = 0;
- makedata();
+ makedata(0, NDISKS-1);
for (ra = raid6_recov_algos; *ra; ra++) {
if ((*ra)->valid && !(*ra)->valid())
continue;
+
raid6_2data_recov = (*ra)->data2;
raid6_datap_recov = (*ra)->datap;
printf("using recovery %s\n", (*ra)->name);
for (algo = raid6_algos; *algo; algo++) {
- if (!(*algo)->valid || (*algo)->valid()) {
- raid6_call = **algo;
+ if ((*algo)->valid && !(*algo)->valid())
+ continue;
+
+ raid6_call = **algo;
+
+ /* Nuke syndromes */
+ memset(data[NDISKS-2], 0xee, 2*PAGE_SIZE);
+
+ /* Generate assumed good syndrome */
+ raid6_call.gen_syndrome(NDISKS, PAGE_SIZE,
+ (void **)&dataptrs);
+
+ for (i = 0; i < NDISKS-1; i++)
+ for (j = i+1; j < NDISKS; j++)
+ err += test_disks(i, j);
+
+ if (!raid6_call.xor_syndrome)
+ continue;
+
+ for (p1 = 0; p1 < NDISKS-2; p1++)
+ for (p2 = p1; p2 < NDISKS-2; p2++) {
- /* Nuke syndromes */
- memset(data[NDISKS-2], 0xee, 2*PAGE_SIZE);
+ /* Simulate rmw run */
+ raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE,
+ (void **)&dataptrs);
+ makedata(p1, p2);
+ raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE,
+ (void **)&dataptrs);
- /* Generate assumed good syndrome */
- raid6_call.gen_syndrome(NDISKS, PAGE_SIZE,
- (void **)&dataptrs);
+ for (i = 0; i < NDISKS-1; i++)
+ for (j = i+1; j < NDISKS; j++)
+ err += test_disks(i, j);
+ }
- for (i = 0; i < NDISKS-1; i++)
- for (j = i+1; j < NDISKS; j++)
- err += test_disks(i, j);
- }
}
printf("\n");
}
diff --git a/lib/raid6/tilegx.uc b/lib/raid6/tilegx.uc
index e7c29459cbcd..2dd291a11264 100644
--- a/lib/raid6/tilegx.uc
+++ b/lib/raid6/tilegx.uc
@@ -80,6 +80,7 @@ void raid6_tilegx$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
const struct raid6_calls raid6_tilegx$# = {
raid6_tilegx$#_gen_syndrome,
+ NULL, /* XOR not yet implemented */
NULL,
"tilegx$#",
0
diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index 4898442b837f..a60a6d335a91 100644
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -14,6 +14,7 @@
* published by the Free Software Foundation.
*/
+#include <linux/atomic.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/log2.h>
@@ -25,6 +26,7 @@
#include <linux/random.h>
#include <linux/rhashtable.h>
#include <linux/err.h>
+#include <linux/export.h>
#define HASH_DEFAULT_SIZE 64UL
#define HASH_MIN_SIZE 4U
@@ -405,13 +407,18 @@ int rhashtable_insert_rehash(struct rhashtable *ht)
if (rht_grow_above_75(ht, tbl))
size *= 2;
- /* More than two rehashes (not resizes) detected. */
- else if (WARN_ON(old_tbl != tbl && old_tbl->size == size))
+ /* Do not schedule more than one rehash */
+ else if (old_tbl != tbl)
return -EBUSY;
new_tbl = bucket_table_alloc(ht, size, GFP_ATOMIC);
- if (new_tbl == NULL)
+ if (new_tbl == NULL) {
+ /* Schedule async resize/rehash to try allocation
+ * non-atomic context.
+ */
+ schedule_work(&ht->run_work);
return -ENOMEM;
+ }
err = rhashtable_rehash_attach(ht, tbl, new_tbl);
if (err) {
@@ -441,6 +448,10 @@ int rhashtable_insert_slow(struct rhashtable *ht, const void *key,
if (key && rhashtable_lookup_fast(ht, key, ht->p))
goto exit;
+ err = -E2BIG;
+ if (unlikely(rht_grow_above_max(ht, tbl)))
+ goto exit;
+
err = -EAGAIN;
if (rhashtable_check_elasticity(ht, tbl, hash) ||
rht_grow_above_100(ht, tbl))
@@ -574,7 +585,6 @@ void *rhashtable_walk_next(struct rhashtable_iter *iter)
struct bucket_table *tbl = iter->walker->tbl;
struct rhashtable *ht = iter->ht;
struct rhash_head *p = iter->p;
- void *obj = NULL;
if (p) {
p = rht_dereference_bucket_rcu(p->next, tbl, iter->slot);
@@ -594,8 +604,7 @@ next:
if (!rht_is_a_nulls(p)) {
iter->skip++;
iter->p = p;
- obj = rht_obj(ht, p);
- goto out;
+ return rht_obj(ht, p);
}
iter->skip = 0;
@@ -613,9 +622,7 @@ next:
iter->p = NULL;
-out:
-
- return obj;
+ return NULL;
}
EXPORT_SYMBOL_GPL(rhashtable_walk_next);
@@ -733,6 +740,12 @@ int rhashtable_init(struct rhashtable *ht,
if (params->max_size)
ht->p.max_size = rounddown_pow_of_two(params->max_size);
+ if (params->insecure_max_entries)
+ ht->p.insecure_max_entries =
+ rounddown_pow_of_two(params->insecure_max_entries);
+ else
+ ht->p.insecure_max_entries = ht->p.max_size * 2;
+
ht->p.min_size = max(ht->p.min_size, HASH_MIN_SIZE);
/* The maximum (not average) chain length grows with the
diff --git a/lib/string.c b/lib/string.c
index a5792019193c..bb3d4b6993c4 100644
--- a/lib/string.c
+++ b/lib/string.c
@@ -607,7 +607,7 @@ EXPORT_SYMBOL(memset);
void memzero_explicit(void *s, size_t count)
{
memset(s, 0, count);
- barrier();
+ barrier_data(s);
}
EXPORT_SYMBOL(memzero_explicit);
diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c
index a28df5206d95..fe9a32591c24 100644
--- a/lib/strnlen_user.c
+++ b/lib/strnlen_user.c
@@ -57,7 +57,8 @@ static inline long do_strnlen_user(const char __user *src, unsigned long count,
return res + find_zero(data) + 1 - align;
}
res += sizeof(unsigned long);
- if (unlikely(max < sizeof(unsigned long)))
+ /* We already handled 'unsigned long' bytes. Did we do it all ? */
+ if (unlikely(max <= sizeof(unsigned long)))
break;
max -= sizeof(unsigned long);
if (unlikely(__get_user(c,(unsigned long __user *)(src+res))))
@@ -89,8 +90,15 @@ static inline long do_strnlen_user(const char __user *src, unsigned long count,
* Get the size of a NUL-terminated string in user space.
*
* Returns the size of the string INCLUDING the terminating NUL.
- * If the string is too long, returns 'count+1'.
+ * If the string is too long, returns a number larger than @count. User
+ * has to check the return value against "> count".
* On exception (or invalid count), returns 0.
+ *
+ * NOTE! You should basically never use this function. There is
+ * almost never any valid case for using the length of a user space
+ * string, since the string can be changed at any time by other
+ * threads. Use "strncpy_from_user()" instead to get a stable copy
+ * of the string.
*/
long strnlen_user(const char __user *str, long count)
{
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 4abda074ea45..3c365ab6cf5f 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -537,8 +537,9 @@ EXPORT_SYMBOL_GPL(swiotlb_tbl_map_single);
* Allocates bounce buffer and returns its kernel virtual address.
*/
-phys_addr_t map_single(struct device *hwdev, phys_addr_t phys, size_t size,
- enum dma_data_direction dir)
+static phys_addr_t
+map_single(struct device *hwdev, phys_addr_t phys, size_t size,
+ enum dma_data_direction dir)
{
dma_addr_t start_dma_addr = phys_to_dma(hwdev, io_tlb_start);
diff --git a/lib/test-hexdump.c b/lib/test-hexdump.c
index 9846ff7428b3..c227cc43ec0a 100644
--- a/lib/test-hexdump.c
+++ b/lib/test-hexdump.c
@@ -48,7 +48,7 @@ static void __init test_hexdump(size_t len, int rowsize, int groupsize,
char test[32 * 3 + 2 + 32 + 1];
char real[32 * 3 + 2 + 32 + 1];
char *p;
- const char **result;
+ const char * const *result;
size_t l = len;
int gs = groupsize, rs = rowsize;
unsigned int i;
diff --git a/lib/test_bpf.c b/lib/test_bpf.c
index 80d78c51f65f..7f58c735d745 100644
--- a/lib/test_bpf.c
+++ b/lib/test_bpf.c
@@ -21,6 +21,7 @@
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/if_vlan.h>
+#include <linux/random.h>
/* General test specific settings */
#define MAX_SUBTESTS 3
@@ -67,6 +68,10 @@ struct bpf_test {
union {
struct sock_filter insns[MAX_INSNS];
struct bpf_insn insns_int[MAX_INSNS];
+ struct {
+ void *insns;
+ unsigned int len;
+ } ptr;
} u;
__u8 aux;
__u8 data[MAX_DATA];
@@ -74,8 +79,282 @@ struct bpf_test {
int data_size;
__u32 result;
} test[MAX_SUBTESTS];
+ int (*fill_helper)(struct bpf_test *self);
};
+/* Large test cases need separate allocation and fill handler. */
+
+static int bpf_fill_maxinsns1(struct bpf_test *self)
+{
+ unsigned int len = BPF_MAXINSNS;
+ struct sock_filter *insn;
+ __u32 k = ~0;
+ int i;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ for (i = 0; i < len; i++, k--)
+ insn[i] = __BPF_STMT(BPF_RET | BPF_K, k);
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = len;
+
+ return 0;
+}
+
+static int bpf_fill_maxinsns2(struct bpf_test *self)
+{
+ unsigned int len = BPF_MAXINSNS;
+ struct sock_filter *insn;
+ int i;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ for (i = 0; i < len; i++)
+ insn[i] = __BPF_STMT(BPF_RET | BPF_K, 0xfefefefe);
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = len;
+
+ return 0;
+}
+
+static int bpf_fill_maxinsns3(struct bpf_test *self)
+{
+ unsigned int len = BPF_MAXINSNS;
+ struct sock_filter *insn;
+ struct rnd_state rnd;
+ int i;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ prandom_seed_state(&rnd, 3141592653589793238ULL);
+
+ for (i = 0; i < len - 1; i++) {
+ __u32 k = prandom_u32_state(&rnd);
+
+ insn[i] = __BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, k);
+ }
+
+ insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0);
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = len;
+
+ return 0;
+}
+
+static int bpf_fill_maxinsns4(struct bpf_test *self)
+{
+ unsigned int len = BPF_MAXINSNS + 1;
+ struct sock_filter *insn;
+ int i;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ for (i = 0; i < len; i++)
+ insn[i] = __BPF_STMT(BPF_RET | BPF_K, 0xfefefefe);
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = len;
+
+ return 0;
+}
+
+static int bpf_fill_maxinsns5(struct bpf_test *self)
+{
+ unsigned int len = BPF_MAXINSNS;
+ struct sock_filter *insn;
+ int i;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ insn[0] = __BPF_JUMP(BPF_JMP | BPF_JA, len - 2, 0, 0);
+
+ for (i = 1; i < len - 1; i++)
+ insn[i] = __BPF_STMT(BPF_RET | BPF_K, 0xfefefefe);
+
+ insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xabababab);
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = len;
+
+ return 0;
+}
+
+static int bpf_fill_maxinsns6(struct bpf_test *self)
+{
+ unsigned int len = BPF_MAXINSNS;
+ struct sock_filter *insn;
+ int i;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ for (i = 0; i < len - 1; i++)
+ insn[i] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF +
+ SKF_AD_VLAN_TAG_PRESENT);
+
+ insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0);
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = len;
+
+ return 0;
+}
+
+static int bpf_fill_maxinsns7(struct bpf_test *self)
+{
+ unsigned int len = BPF_MAXINSNS;
+ struct sock_filter *insn;
+ int i;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ for (i = 0; i < len - 4; i++)
+ insn[i] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF +
+ SKF_AD_CPU);
+
+ insn[len - 4] = __BPF_STMT(BPF_MISC | BPF_TAX, 0);
+ insn[len - 3] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF +
+ SKF_AD_CPU);
+ insn[len - 2] = __BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0);
+ insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0);
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = len;
+
+ return 0;
+}
+
+static int bpf_fill_maxinsns8(struct bpf_test *self)
+{
+ unsigned int len = BPF_MAXINSNS;
+ struct sock_filter *insn;
+ int i, jmp_off = len - 3;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ insn[0] = __BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff);
+
+ for (i = 1; i < len - 1; i++)
+ insn[i] = __BPF_JUMP(BPF_JMP | BPF_JGT, 0xffffffff, jmp_off--, 0);
+
+ insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0);
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = len;
+
+ return 0;
+}
+
+static int bpf_fill_maxinsns9(struct bpf_test *self)
+{
+ unsigned int len = BPF_MAXINSNS;
+ struct bpf_insn *insn;
+ int i;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ insn[0] = BPF_JMP_IMM(BPF_JA, 0, 0, len - 2);
+ insn[1] = BPF_ALU32_IMM(BPF_MOV, R0, 0xcbababab);
+ insn[2] = BPF_EXIT_INSN();
+
+ for (i = 3; i < len - 2; i++)
+ insn[i] = BPF_ALU32_IMM(BPF_MOV, R0, 0xfefefefe);
+
+ insn[len - 2] = BPF_EXIT_INSN();
+ insn[len - 1] = BPF_JMP_IMM(BPF_JA, 0, 0, -(len - 1));
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = len;
+
+ return 0;
+}
+
+static int bpf_fill_maxinsns10(struct bpf_test *self)
+{
+ unsigned int len = BPF_MAXINSNS, hlen = len - 2;
+ struct bpf_insn *insn;
+ int i;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ for (i = 0; i < hlen / 2; i++)
+ insn[i] = BPF_JMP_IMM(BPF_JA, 0, 0, hlen - 2 - 2 * i);
+ for (i = hlen - 1; i > hlen / 2; i--)
+ insn[i] = BPF_JMP_IMM(BPF_JA, 0, 0, hlen - 1 - 2 * i);
+
+ insn[hlen / 2] = BPF_JMP_IMM(BPF_JA, 0, 0, hlen / 2 - 1);
+ insn[hlen] = BPF_ALU32_IMM(BPF_MOV, R0, 0xabababac);
+ insn[hlen + 1] = BPF_EXIT_INSN();
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = len;
+
+ return 0;
+}
+
+static int __bpf_fill_ja(struct bpf_test *self, unsigned int len,
+ unsigned int plen)
+{
+ struct sock_filter *insn;
+ unsigned int rlen;
+ int i, j;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ rlen = (len % plen) - 1;
+
+ for (i = 0; i + plen < len; i += plen)
+ for (j = 0; j < plen; j++)
+ insn[i + j] = __BPF_JUMP(BPF_JMP | BPF_JA,
+ plen - 1 - j, 0, 0);
+ for (j = 0; j < rlen; j++)
+ insn[i + j] = __BPF_JUMP(BPF_JMP | BPF_JA, rlen - 1 - j,
+ 0, 0);
+
+ insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xababcbac);
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = len;
+
+ return 0;
+}
+
+static int bpf_fill_maxinsns11(struct bpf_test *self)
+{
+ /* Hits 70 passes on x86_64, so cannot get JITed there. */
+ return __bpf_fill_ja(self, BPF_MAXINSNS, 68);
+}
+
+static int bpf_fill_ja(struct bpf_test *self)
+{
+ /* Hits exactly 11 passes on x86_64 JIT. */
+ return __bpf_fill_ja(self, 12, 9);
+}
+
static struct bpf_test tests[] = {
{
"TAX",
@@ -1755,7 +2034,8 @@ static struct bpf_test tests[] = {
BPF_EXIT_INSN(),
BPF_JMP_IMM(BPF_JEQ, R3, 0x1234, 1),
BPF_EXIT_INSN(),
- BPF_ALU64_IMM(BPF_MOV, R0, 1),
+ BPF_LD_IMM64(R0, 0x1ffffffffLL),
+ BPF_ALU64_IMM(BPF_RSH, R0, 32), /* R0 = 1 */
BPF_EXIT_INSN(),
},
INTERNAL,
@@ -1805,6 +2085,2313 @@ static struct bpf_test tests[] = {
0x10, 0xbf, 0x48, 0xd6, 0x43, 0xd6},
{ { 38, 256 } }
},
+ /* BPF_ALU | BPF_MOV | BPF_X */
+ {
+ "ALU_MOV_X: dst = 2",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R1, 2),
+ BPF_ALU32_REG(BPF_MOV, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU_MOV_X: dst = 4294967295",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R1, 4294967295U),
+ BPF_ALU32_REG(BPF_MOV, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 4294967295U } },
+ },
+ {
+ "ALU64_MOV_X: dst = 2",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R1, 2),
+ BPF_ALU64_REG(BPF_MOV, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU64_MOV_X: dst = 4294967295",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R1, 4294967295U),
+ BPF_ALU64_REG(BPF_MOV, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 4294967295U } },
+ },
+ /* BPF_ALU | BPF_MOV | BPF_K */
+ {
+ "ALU_MOV_K: dst = 2",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 2),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU_MOV_K: dst = 4294967295",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 4294967295U),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 4294967295U } },
+ },
+ {
+ "ALU_MOV_K: 0x0000ffffffff0000 = 0x00000000ffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0000ffffffff0000LL),
+ BPF_LD_IMM64(R3, 0x00000000ffffffffLL),
+ BPF_ALU32_IMM(BPF_MOV, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ALU64_MOV_K: dst = 2",
+ .u.insns_int = {
+ BPF_ALU64_IMM(BPF_MOV, R0, 2),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU64_MOV_K: dst = 2147483647",
+ .u.insns_int = {
+ BPF_ALU64_IMM(BPF_MOV, R0, 2147483647),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2147483647 } },
+ },
+ {
+ "ALU64_OR_K: dst = 0x0",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0000ffffffff0000LL),
+ BPF_LD_IMM64(R3, 0x0),
+ BPF_ALU64_IMM(BPF_MOV, R2, 0x0),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ALU64_MOV_K: dst = -1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0000ffffffff0000LL),
+ BPF_LD_IMM64(R3, 0xffffffffffffffffLL),
+ BPF_ALU64_IMM(BPF_MOV, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ /* BPF_ALU | BPF_ADD | BPF_X */
+ {
+ "ALU_ADD_X: 1 + 2 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2),
+ BPF_ALU32_REG(BPF_ADD, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU_ADD_X: 1 + 4294967294 = 4294967295",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U),
+ BPF_ALU32_REG(BPF_ADD, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 4294967295U } },
+ },
+ {
+ "ALU64_ADD_X: 1 + 2 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2),
+ BPF_ALU64_REG(BPF_ADD, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_ADD_X: 1 + 4294967294 = 4294967295",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U),
+ BPF_ALU64_REG(BPF_ADD, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 4294967295U } },
+ },
+ /* BPF_ALU | BPF_ADD | BPF_K */
+ {
+ "ALU_ADD_K: 1 + 2 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_ADD, R0, 2),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU_ADD_K: 3 + 0 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_ADD, R0, 0),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU_ADD_K: 1 + 4294967294 = 4294967295",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_ADD, R0, 4294967294U),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 4294967295U } },
+ },
+ {
+ "ALU_ADD_K: 0 + (-1) = 0x00000000ffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0),
+ BPF_LD_IMM64(R3, 0x00000000ffffffff),
+ BPF_ALU32_IMM(BPF_ADD, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ALU64_ADD_K: 1 + 2 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU64_IMM(BPF_ADD, R0, 2),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_ADD_K: 3 + 0 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU64_IMM(BPF_ADD, R0, 0),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_ADD_K: 1 + 2147483646 = 2147483647",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU64_IMM(BPF_ADD, R0, 2147483646),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2147483647 } },
+ },
+ {
+ "ALU64_ADD_K: 2147483646 + -2147483647 = -1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2147483646),
+ BPF_ALU64_IMM(BPF_ADD, R0, -2147483647),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, -1 } },
+ },
+ {
+ "ALU64_ADD_K: 1 + 0 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x1),
+ BPF_LD_IMM64(R3, 0x1),
+ BPF_ALU64_IMM(BPF_ADD, R2, 0x0),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ALU64_ADD_K: 0 + (-1) = 0xffffffffffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0),
+ BPF_LD_IMM64(R3, 0xffffffffffffffffLL),
+ BPF_ALU64_IMM(BPF_ADD, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ /* BPF_ALU | BPF_SUB | BPF_X */
+ {
+ "ALU_SUB_X: 3 - 1 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_MOV, R1, 1),
+ BPF_ALU32_REG(BPF_SUB, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU_SUB_X: 4294967295 - 4294967294 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 4294967295U),
+ BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U),
+ BPF_ALU32_REG(BPF_SUB, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU64_SUB_X: 3 - 1 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_MOV, R1, 1),
+ BPF_ALU64_REG(BPF_SUB, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU64_SUB_X: 4294967295 - 4294967294 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 4294967295U),
+ BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U),
+ BPF_ALU64_REG(BPF_SUB, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_ALU | BPF_SUB | BPF_K */
+ {
+ "ALU_SUB_K: 3 - 1 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_SUB, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU_SUB_K: 3 - 0 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_SUB, R0, 0),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU_SUB_K: 4294967295 - 4294967294 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 4294967295U),
+ BPF_ALU32_IMM(BPF_SUB, R0, 4294967294U),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU64_SUB_K: 3 - 1 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU64_IMM(BPF_SUB, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU64_SUB_K: 3 - 0 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU64_IMM(BPF_SUB, R0, 0),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_SUB_K: 4294967294 - 4294967295 = -1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 4294967294U),
+ BPF_ALU64_IMM(BPF_SUB, R0, 4294967295U),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, -1 } },
+ },
+ {
+ "ALU64_ADD_K: 2147483646 - 2147483647 = -1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2147483646),
+ BPF_ALU64_IMM(BPF_SUB, R0, 2147483647),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, -1 } },
+ },
+ /* BPF_ALU | BPF_MUL | BPF_X */
+ {
+ "ALU_MUL_X: 2 * 3 = 6",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2),
+ BPF_ALU32_IMM(BPF_MOV, R1, 3),
+ BPF_ALU32_REG(BPF_MUL, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 6 } },
+ },
+ {
+ "ALU_MUL_X: 2 * 0x7FFFFFF8 = 0xFFFFFFF0",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2),
+ BPF_ALU32_IMM(BPF_MOV, R1, 0x7FFFFFF8),
+ BPF_ALU32_REG(BPF_MUL, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xFFFFFFF0 } },
+ },
+ {
+ "ALU_MUL_X: -1 * -1 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, -1),
+ BPF_ALU32_IMM(BPF_MOV, R1, -1),
+ BPF_ALU32_REG(BPF_MUL, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU64_MUL_X: 2 * 3 = 6",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2),
+ BPF_ALU32_IMM(BPF_MOV, R1, 3),
+ BPF_ALU64_REG(BPF_MUL, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 6 } },
+ },
+ {
+ "ALU64_MUL_X: 1 * 2147483647 = 2147483647",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2147483647),
+ BPF_ALU64_REG(BPF_MUL, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2147483647 } },
+ },
+ /* BPF_ALU | BPF_MUL | BPF_K */
+ {
+ "ALU_MUL_K: 2 * 3 = 6",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2),
+ BPF_ALU32_IMM(BPF_MUL, R0, 3),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 6 } },
+ },
+ {
+ "ALU_MUL_K: 3 * 1 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_MUL, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU_MUL_K: 2 * 0x7FFFFFF8 = 0xFFFFFFF0",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2),
+ BPF_ALU32_IMM(BPF_MUL, R0, 0x7FFFFFF8),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xFFFFFFF0 } },
+ },
+ {
+ "ALU_MUL_K: 1 * (-1) = 0x00000000ffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x1),
+ BPF_LD_IMM64(R3, 0x00000000ffffffff),
+ BPF_ALU32_IMM(BPF_MUL, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ALU64_MUL_K: 2 * 3 = 6",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2),
+ BPF_ALU64_IMM(BPF_MUL, R0, 3),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 6 } },
+ },
+ {
+ "ALU64_MUL_K: 3 * 1 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU64_IMM(BPF_MUL, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_MUL_K: 1 * 2147483647 = 2147483647",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU64_IMM(BPF_MUL, R0, 2147483647),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2147483647 } },
+ },
+ {
+ "ALU64_MUL_K: 1 * -2147483647 = -2147483647",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU64_IMM(BPF_MUL, R0, -2147483647),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, -2147483647 } },
+ },
+ {
+ "ALU64_MUL_K: 1 * (-1) = 0xffffffffffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x1),
+ BPF_LD_IMM64(R3, 0xffffffffffffffffLL),
+ BPF_ALU64_IMM(BPF_MUL, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ /* BPF_ALU | BPF_DIV | BPF_X */
+ {
+ "ALU_DIV_X: 6 / 2 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 6),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2),
+ BPF_ALU32_REG(BPF_DIV, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU_DIV_X: 4294967295 / 4294967295 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 4294967295U),
+ BPF_ALU32_IMM(BPF_MOV, R1, 4294967295U),
+ BPF_ALU32_REG(BPF_DIV, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU64_DIV_X: 6 / 2 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 6),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2),
+ BPF_ALU64_REG(BPF_DIV, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_DIV_X: 2147483647 / 2147483647 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2147483647),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2147483647),
+ BPF_ALU64_REG(BPF_DIV, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU64_DIV_X: 0xffffffffffffffff / (-1) = 0x0000000000000001",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0xffffffffffffffffLL),
+ BPF_LD_IMM64(R4, 0xffffffffffffffffLL),
+ BPF_LD_IMM64(R3, 0x0000000000000001LL),
+ BPF_ALU64_REG(BPF_DIV, R2, R4),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ /* BPF_ALU | BPF_DIV | BPF_K */
+ {
+ "ALU_DIV_K: 6 / 2 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 6),
+ BPF_ALU32_IMM(BPF_DIV, R0, 2),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU_DIV_K: 3 / 1 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_DIV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU_DIV_K: 4294967295 / 4294967295 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 4294967295U),
+ BPF_ALU32_IMM(BPF_DIV, R0, 4294967295U),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU_DIV_K: 0xffffffffffffffff / (-1) = 0x1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0xffffffffffffffffLL),
+ BPF_LD_IMM64(R3, 0x1UL),
+ BPF_ALU32_IMM(BPF_DIV, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ALU64_DIV_K: 6 / 2 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 6),
+ BPF_ALU64_IMM(BPF_DIV, R0, 2),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_DIV_K: 3 / 1 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU64_IMM(BPF_DIV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_DIV_K: 2147483647 / 2147483647 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2147483647),
+ BPF_ALU64_IMM(BPF_DIV, R0, 2147483647),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU64_DIV_K: 0xffffffffffffffff / (-1) = 0x0000000000000001",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0xffffffffffffffffLL),
+ BPF_LD_IMM64(R3, 0x0000000000000001LL),
+ BPF_ALU64_IMM(BPF_DIV, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ /* BPF_ALU | BPF_MOD | BPF_X */
+ {
+ "ALU_MOD_X: 3 % 2 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2),
+ BPF_ALU32_REG(BPF_MOD, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU_MOD_X: 4294967295 % 4294967293 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 4294967295U),
+ BPF_ALU32_IMM(BPF_MOV, R1, 4294967293U),
+ BPF_ALU32_REG(BPF_MOD, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU64_MOD_X: 3 % 2 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2),
+ BPF_ALU64_REG(BPF_MOD, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU64_MOD_X: 2147483647 % 2147483645 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2147483647),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2147483645),
+ BPF_ALU64_REG(BPF_MOD, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ /* BPF_ALU | BPF_MOD | BPF_K */
+ {
+ "ALU_MOD_K: 3 % 2 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_MOD, R0, 2),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU_MOD_K: 3 % 1 = 0",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_MOD, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0 } },
+ },
+ {
+ "ALU_MOD_K: 4294967295 % 4294967293 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 4294967295U),
+ BPF_ALU32_IMM(BPF_MOD, R0, 4294967293U),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU64_MOD_K: 3 % 2 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU64_IMM(BPF_MOD, R0, 2),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU64_MOD_K: 3 % 1 = 0",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU64_IMM(BPF_MOD, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0 } },
+ },
+ {
+ "ALU64_MOD_K: 2147483647 % 2147483645 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2147483647),
+ BPF_ALU64_IMM(BPF_MOD, R0, 2147483645),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ /* BPF_ALU | BPF_AND | BPF_X */
+ {
+ "ALU_AND_X: 3 & 2 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2),
+ BPF_ALU32_REG(BPF_AND, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU_AND_X: 0xffffffff & 0xffffffff = 0xffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0xffffffff),
+ BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff),
+ BPF_ALU32_REG(BPF_AND, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffff } },
+ },
+ {
+ "ALU64_AND_X: 3 & 2 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2),
+ BPF_ALU64_REG(BPF_AND, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU64_AND_X: 0xffffffff & 0xffffffff = 0xffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0xffffffff),
+ BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff),
+ BPF_ALU64_REG(BPF_AND, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffff } },
+ },
+ /* BPF_ALU | BPF_AND | BPF_K */
+ {
+ "ALU_AND_K: 3 & 2 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU32_IMM(BPF_AND, R0, 2),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU_AND_K: 0xffffffff & 0xffffffff = 0xffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0xffffffff),
+ BPF_ALU32_IMM(BPF_AND, R0, 0xffffffff),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffff } },
+ },
+ {
+ "ALU64_AND_K: 3 & 2 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU64_IMM(BPF_AND, R0, 2),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU64_AND_K: 0xffffffff & 0xffffffff = 0xffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0xffffffff),
+ BPF_ALU64_IMM(BPF_AND, R0, 0xffffffff),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffff } },
+ },
+ {
+ "ALU64_AND_K: 0x0000ffffffff0000 & 0x0 = 0x0000ffff00000000",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0000ffffffff0000LL),
+ BPF_LD_IMM64(R3, 0x0000000000000000LL),
+ BPF_ALU64_IMM(BPF_AND, R2, 0x0),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ALU64_AND_K: 0x0000ffffffff0000 & -1 = 0x0000ffffffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0000ffffffff0000LL),
+ BPF_LD_IMM64(R3, 0x0000ffffffff0000LL),
+ BPF_ALU64_IMM(BPF_AND, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ALU64_AND_K: 0xffffffffffffffff & -1 = 0xffffffffffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0xffffffffffffffffLL),
+ BPF_LD_IMM64(R3, 0xffffffffffffffffLL),
+ BPF_ALU64_IMM(BPF_AND, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ /* BPF_ALU | BPF_OR | BPF_X */
+ {
+ "ALU_OR_X: 1 | 2 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2),
+ BPF_ALU32_REG(BPF_OR, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU_OR_X: 0x0 | 0xffffffff = 0xffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0),
+ BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff),
+ BPF_ALU32_REG(BPF_OR, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffff } },
+ },
+ {
+ "ALU64_OR_X: 1 | 2 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 2),
+ BPF_ALU64_REG(BPF_OR, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_OR_X: 0 | 0xffffffff = 0xffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0),
+ BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff),
+ BPF_ALU64_REG(BPF_OR, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffff } },
+ },
+ /* BPF_ALU | BPF_OR | BPF_K */
+ {
+ "ALU_OR_K: 1 | 2 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_OR, R0, 2),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU_OR_K: 0 & 0xffffffff = 0xffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0),
+ BPF_ALU32_IMM(BPF_OR, R0, 0xffffffff),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffff } },
+ },
+ {
+ "ALU64_OR_K: 1 | 2 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU64_IMM(BPF_OR, R0, 2),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_OR_K: 0 & 0xffffffff = 0xffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0),
+ BPF_ALU64_IMM(BPF_OR, R0, 0xffffffff),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffff } },
+ },
+ {
+ "ALU64_OR_K: 0x0000ffffffff0000 | 0x0 = 0x0000ffff00000000",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0000ffffffff0000LL),
+ BPF_LD_IMM64(R3, 0x0000ffffffff0000LL),
+ BPF_ALU64_IMM(BPF_OR, R2, 0x0),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ALU64_OR_K: 0x0000ffffffff0000 | -1 = 0xffffffffffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0000ffffffff0000LL),
+ BPF_LD_IMM64(R3, 0xffffffffffffffffLL),
+ BPF_ALU64_IMM(BPF_OR, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ALU64_OR_K: 0x000000000000000 | -1 = 0xffffffffffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0000000000000000LL),
+ BPF_LD_IMM64(R3, 0xffffffffffffffffLL),
+ BPF_ALU64_IMM(BPF_OR, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ /* BPF_ALU | BPF_XOR | BPF_X */
+ {
+ "ALU_XOR_X: 5 ^ 6 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 5),
+ BPF_ALU32_IMM(BPF_MOV, R1, 6),
+ BPF_ALU32_REG(BPF_XOR, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU_XOR_X: 0x1 ^ 0xffffffff = 0xfffffffe",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff),
+ BPF_ALU32_REG(BPF_XOR, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xfffffffe } },
+ },
+ {
+ "ALU64_XOR_X: 5 ^ 6 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 5),
+ BPF_ALU32_IMM(BPF_MOV, R1, 6),
+ BPF_ALU64_REG(BPF_XOR, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_XOR_X: 1 ^ 0xffffffff = 0xfffffffe",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff),
+ BPF_ALU64_REG(BPF_XOR, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xfffffffe } },
+ },
+ /* BPF_ALU | BPF_XOR | BPF_K */
+ {
+ "ALU_XOR_K: 5 ^ 6 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 5),
+ BPF_ALU32_IMM(BPF_XOR, R0, 6),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU_XOR_K: 1 ^ 0xffffffff = 0xfffffffe",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_XOR, R0, 0xffffffff),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xfffffffe } },
+ },
+ {
+ "ALU64_XOR_K: 5 ^ 6 = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 5),
+ BPF_ALU64_IMM(BPF_XOR, R0, 6),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_XOR_K: 1 & 0xffffffff = 0xfffffffe",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU64_IMM(BPF_XOR, R0, 0xffffffff),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xfffffffe } },
+ },
+ {
+ "ALU64_XOR_K: 0x0000ffffffff0000 ^ 0x0 = 0x0000ffffffff0000",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0000ffffffff0000LL),
+ BPF_LD_IMM64(R3, 0x0000ffffffff0000LL),
+ BPF_ALU64_IMM(BPF_XOR, R2, 0x0),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ALU64_XOR_K: 0x0000ffffffff0000 ^ -1 = 0xffff00000000ffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0000ffffffff0000LL),
+ BPF_LD_IMM64(R3, 0xffff00000000ffffLL),
+ BPF_ALU64_IMM(BPF_XOR, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ALU64_XOR_K: 0x000000000000000 ^ -1 = 0xffffffffffffffff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0x0000000000000000LL),
+ BPF_LD_IMM64(R3, 0xffffffffffffffffLL),
+ BPF_ALU64_IMM(BPF_XOR, R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ /* BPF_ALU | BPF_LSH | BPF_X */
+ {
+ "ALU_LSH_X: 1 << 1 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 1),
+ BPF_ALU32_REG(BPF_LSH, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU_LSH_X: 1 << 31 = 0x80000000",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 31),
+ BPF_ALU32_REG(BPF_LSH, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x80000000 } },
+ },
+ {
+ "ALU64_LSH_X: 1 << 1 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 1),
+ BPF_ALU64_REG(BPF_LSH, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU64_LSH_X: 1 << 31 = 0x80000000",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_MOV, R1, 31),
+ BPF_ALU64_REG(BPF_LSH, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x80000000 } },
+ },
+ /* BPF_ALU | BPF_LSH | BPF_K */
+ {
+ "ALU_LSH_K: 1 << 1 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_LSH, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU_LSH_K: 1 << 31 = 0x80000000",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU32_IMM(BPF_LSH, R0, 31),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x80000000 } },
+ },
+ {
+ "ALU64_LSH_K: 1 << 1 = 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU64_IMM(BPF_LSH, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 2 } },
+ },
+ {
+ "ALU64_LSH_K: 1 << 31 = 0x80000000",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 1),
+ BPF_ALU64_IMM(BPF_LSH, R0, 31),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x80000000 } },
+ },
+ /* BPF_ALU | BPF_RSH | BPF_X */
+ {
+ "ALU_RSH_X: 2 >> 1 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2),
+ BPF_ALU32_IMM(BPF_MOV, R1, 1),
+ BPF_ALU32_REG(BPF_RSH, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU_RSH_X: 0x80000000 >> 31 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0x80000000),
+ BPF_ALU32_IMM(BPF_MOV, R1, 31),
+ BPF_ALU32_REG(BPF_RSH, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU64_RSH_X: 2 >> 1 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2),
+ BPF_ALU32_IMM(BPF_MOV, R1, 1),
+ BPF_ALU64_REG(BPF_RSH, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU64_RSH_X: 0x80000000 >> 31 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0x80000000),
+ BPF_ALU32_IMM(BPF_MOV, R1, 31),
+ BPF_ALU64_REG(BPF_RSH, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_ALU | BPF_RSH | BPF_K */
+ {
+ "ALU_RSH_K: 2 >> 1 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2),
+ BPF_ALU32_IMM(BPF_RSH, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU_RSH_K: 0x80000000 >> 31 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0x80000000),
+ BPF_ALU32_IMM(BPF_RSH, R0, 31),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU64_RSH_K: 2 >> 1 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 2),
+ BPF_ALU64_IMM(BPF_RSH, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "ALU64_RSH_K: 0x80000000 >> 31 = 1",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0x80000000),
+ BPF_ALU64_IMM(BPF_RSH, R0, 31),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_ALU | BPF_ARSH | BPF_X */
+ {
+ "ALU_ARSH_X: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0xff00ff0000000000LL),
+ BPF_ALU32_IMM(BPF_MOV, R1, 40),
+ BPF_ALU64_REG(BPF_ARSH, R0, R1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffff00ff } },
+ },
+ /* BPF_ALU | BPF_ARSH | BPF_K */
+ {
+ "ALU_ARSH_K: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0xff00ff0000000000LL),
+ BPF_ALU64_IMM(BPF_ARSH, R0, 40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffff00ff } },
+ },
+ /* BPF_ALU | BPF_NEG */
+ {
+ "ALU_NEG: -(3) = -3",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 3),
+ BPF_ALU32_IMM(BPF_NEG, R0, 0),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, -3 } },
+ },
+ {
+ "ALU_NEG: -(-3) = 3",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, -3),
+ BPF_ALU32_IMM(BPF_NEG, R0, 0),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ {
+ "ALU64_NEG: -(3) = -3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 3),
+ BPF_ALU64_IMM(BPF_NEG, R0, 0),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, -3 } },
+ },
+ {
+ "ALU64_NEG: -(-3) = 3",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, -3),
+ BPF_ALU64_IMM(BPF_NEG, R0, 0),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 3 } },
+ },
+ /* BPF_ALU | BPF_END | BPF_FROM_BE */
+ {
+ "ALU_END_FROM_BE 16: 0x0123456789abcdef -> 0xcdef",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0x0123456789abcdefLL),
+ BPF_ENDIAN(BPF_FROM_BE, R0, 16),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, cpu_to_be16(0xcdef) } },
+ },
+ {
+ "ALU_END_FROM_BE 32: 0x0123456789abcdef -> 0x89abcdef",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0x0123456789abcdefLL),
+ BPF_ENDIAN(BPF_FROM_BE, R0, 32),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, cpu_to_be32(0x89abcdef) } },
+ },
+ {
+ "ALU_END_FROM_BE 64: 0x0123456789abcdef -> 0x89abcdef",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0x0123456789abcdefLL),
+ BPF_ENDIAN(BPF_FROM_BE, R0, 64),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, (u32) cpu_to_be64(0x0123456789abcdefLL) } },
+ },
+ /* BPF_ALU | BPF_END | BPF_FROM_LE */
+ {
+ "ALU_END_FROM_LE 16: 0x0123456789abcdef -> 0xefcd",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0x0123456789abcdefLL),
+ BPF_ENDIAN(BPF_FROM_LE, R0, 16),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, cpu_to_le16(0xcdef) } },
+ },
+ {
+ "ALU_END_FROM_LE 32: 0x0123456789abcdef -> 0xefcdab89",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0x0123456789abcdefLL),
+ BPF_ENDIAN(BPF_FROM_LE, R0, 32),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, cpu_to_le32(0x89abcdef) } },
+ },
+ {
+ "ALU_END_FROM_LE 64: 0x0123456789abcdef -> 0x67452301",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0x0123456789abcdefLL),
+ BPF_ENDIAN(BPF_FROM_LE, R0, 64),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, (u32) cpu_to_le64(0x0123456789abcdefLL) } },
+ },
+ /* BPF_ST(X) | BPF_MEM | BPF_B/H/W/DW */
+ {
+ "ST_MEM_B: Store/Load byte: max negative",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_ST_MEM(BPF_B, R10, -40, 0xff),
+ BPF_LDX_MEM(BPF_B, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xff } },
+ },
+ {
+ "ST_MEM_B: Store/Load byte: max positive",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_ST_MEM(BPF_H, R10, -40, 0x7f),
+ BPF_LDX_MEM(BPF_H, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x7f } },
+ },
+ {
+ "STX_MEM_B: Store/Load byte: max negative",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0),
+ BPF_LD_IMM64(R1, 0xffLL),
+ BPF_STX_MEM(BPF_B, R10, R1, -40),
+ BPF_LDX_MEM(BPF_B, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xff } },
+ },
+ {
+ "ST_MEM_H: Store/Load half word: max negative",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_ST_MEM(BPF_H, R10, -40, 0xffff),
+ BPF_LDX_MEM(BPF_H, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffff } },
+ },
+ {
+ "ST_MEM_H: Store/Load half word: max positive",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_ST_MEM(BPF_H, R10, -40, 0x7fff),
+ BPF_LDX_MEM(BPF_H, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x7fff } },
+ },
+ {
+ "STX_MEM_H: Store/Load half word: max negative",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0),
+ BPF_LD_IMM64(R1, 0xffffLL),
+ BPF_STX_MEM(BPF_H, R10, R1, -40),
+ BPF_LDX_MEM(BPF_H, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffff } },
+ },
+ {
+ "ST_MEM_W: Store/Load word: max negative",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_ST_MEM(BPF_W, R10, -40, 0xffffffff),
+ BPF_LDX_MEM(BPF_W, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffff } },
+ },
+ {
+ "ST_MEM_W: Store/Load word: max positive",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_ST_MEM(BPF_W, R10, -40, 0x7fffffff),
+ BPF_LDX_MEM(BPF_W, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x7fffffff } },
+ },
+ {
+ "STX_MEM_W: Store/Load word: max negative",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0),
+ BPF_LD_IMM64(R1, 0xffffffffLL),
+ BPF_STX_MEM(BPF_W, R10, R1, -40),
+ BPF_LDX_MEM(BPF_W, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffff } },
+ },
+ {
+ "ST_MEM_DW: Store/Load double word: max negative",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_ST_MEM(BPF_DW, R10, -40, 0xffffffff),
+ BPF_LDX_MEM(BPF_DW, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffff } },
+ },
+ {
+ "ST_MEM_DW: Store/Load double word: max negative 2",
+ .u.insns_int = {
+ BPF_LD_IMM64(R2, 0xffff00000000ffffLL),
+ BPF_LD_IMM64(R3, 0xffffffffffffffffLL),
+ BPF_ST_MEM(BPF_DW, R10, -40, 0xffffffff),
+ BPF_LDX_MEM(BPF_DW, R2, R10, -40),
+ BPF_JMP_REG(BPF_JEQ, R2, R3, 2),
+ BPF_MOV32_IMM(R0, 2),
+ BPF_EXIT_INSN(),
+ BPF_MOV32_IMM(R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x1 } },
+ },
+ {
+ "ST_MEM_DW: Store/Load double word: max positive",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_ST_MEM(BPF_DW, R10, -40, 0x7fffffff),
+ BPF_LDX_MEM(BPF_DW, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x7fffffff } },
+ },
+ {
+ "STX_MEM_DW: Store/Load double word: max negative",
+ .u.insns_int = {
+ BPF_LD_IMM64(R0, 0),
+ BPF_LD_IMM64(R1, 0xffffffffffffffffLL),
+ BPF_STX_MEM(BPF_W, R10, R1, -40),
+ BPF_LDX_MEM(BPF_W, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffff } },
+ },
+ /* BPF_STX | BPF_XADD | BPF_W/DW */
+ {
+ "STX_XADD_W: Test: 0x12 + 0x10 = 0x22",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
+ BPF_ST_MEM(BPF_W, R10, -40, 0x10),
+ BPF_STX_XADD(BPF_W, R10, R0, -40),
+ BPF_LDX_MEM(BPF_W, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x22 } },
+ },
+ {
+ "STX_XADD_DW: Test: 0x12 + 0x10 = 0x22",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
+ BPF_ST_MEM(BPF_DW, R10, -40, 0x10),
+ BPF_STX_XADD(BPF_DW, R10, R0, -40),
+ BPF_LDX_MEM(BPF_DW, R0, R10, -40),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x22 } },
+ },
+ /* BPF_JMP | BPF_EXIT */
+ {
+ "JMP_EXIT",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0x4711),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 0x4712),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0x4711 } },
+ },
+ /* BPF_JMP | BPF_JA */
+ {
+ "JMP_JA: Unconditional jump: if (true) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JSGT | BPF_K */
+ {
+ "JMP_JSGT_K: Signed jump: if (-1 > -2) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 0xffffffffffffffffLL),
+ BPF_JMP_IMM(BPF_JSGT, R1, -2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JSGT_K: Signed jump: if (-1 > -1) return 0",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_LD_IMM64(R1, 0xffffffffffffffffLL),
+ BPF_JMP_IMM(BPF_JSGT, R1, -1, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JSGE | BPF_K */
+ {
+ "JMP_JSGE_K: Signed jump: if (-1 >= -2) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 0xffffffffffffffffLL),
+ BPF_JMP_IMM(BPF_JSGE, R1, -2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JSGE_K: Signed jump: if (-1 >= -1) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 0xffffffffffffffffLL),
+ BPF_JMP_IMM(BPF_JSGE, R1, -1, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JGT | BPF_K */
+ {
+ "JMP_JGT_K: if (3 > 2) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_JMP_IMM(BPF_JGT, R1, 2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JGE | BPF_K */
+ {
+ "JMP_JGE_K: if (3 >= 2) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_JMP_IMM(BPF_JGE, R1, 2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JGT | BPF_K jump backwards */
+ {
+ "JMP_JGT_K: if (3 > 2) return 1 (jump backwards)",
+ .u.insns_int = {
+ BPF_JMP_IMM(BPF_JA, 0, 0, 2), /* goto start */
+ BPF_ALU32_IMM(BPF_MOV, R0, 1), /* out: */
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 0), /* start: */
+ BPF_LD_IMM64(R1, 3), /* note: this takes 2 insns */
+ BPF_JMP_IMM(BPF_JGT, R1, 2, -6), /* goto out */
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JGE_K: if (3 >= 3) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_JMP_IMM(BPF_JGE, R1, 3, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JNE | BPF_K */
+ {
+ "JMP_JNE_K: if (3 != 2) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_JMP_IMM(BPF_JNE, R1, 2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JEQ | BPF_K */
+ {
+ "JMP_JEQ_K: if (3 == 3) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_JMP_IMM(BPF_JEQ, R1, 3, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JSET | BPF_K */
+ {
+ "JMP_JSET_K: if (0x3 & 0x2) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_JMP_IMM(BPF_JNE, R1, 2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JSET_K: if (0x3 & 0xffffffff) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_JMP_IMM(BPF_JNE, R1, 0xffffffff, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JSGT | BPF_X */
+ {
+ "JMP_JSGT_X: Signed jump: if (-1 > -2) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, -1),
+ BPF_LD_IMM64(R2, -2),
+ BPF_JMP_REG(BPF_JSGT, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JSGT_X: Signed jump: if (-1 > -1) return 0",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_LD_IMM64(R1, -1),
+ BPF_LD_IMM64(R2, -1),
+ BPF_JMP_REG(BPF_JSGT, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JSGE | BPF_X */
+ {
+ "JMP_JSGE_X: Signed jump: if (-1 >= -2) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, -1),
+ BPF_LD_IMM64(R2, -2),
+ BPF_JMP_REG(BPF_JSGE, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JSGE_X: Signed jump: if (-1 >= -1) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, -1),
+ BPF_LD_IMM64(R2, -1),
+ BPF_JMP_REG(BPF_JSGE, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JGT | BPF_X */
+ {
+ "JMP_JGT_X: if (3 > 2) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 2),
+ BPF_JMP_REG(BPF_JGT, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JGE | BPF_X */
+ {
+ "JMP_JGE_X: if (3 >= 2) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 2),
+ BPF_JMP_REG(BPF_JGE, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JGE_X: if (3 >= 3) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 3),
+ BPF_JMP_REG(BPF_JGE, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JNE | BPF_X */
+ {
+ "JMP_JNE_X: if (3 != 2) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 2),
+ BPF_JMP_REG(BPF_JNE, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JEQ | BPF_X */
+ {
+ "JMP_JEQ_X: if (3 == 3) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 3),
+ BPF_JMP_REG(BPF_JEQ, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ /* BPF_JMP | BPF_JSET | BPF_X */
+ {
+ "JMP_JSET_X: if (0x3 & 0x2) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 2),
+ BPF_JMP_REG(BPF_JNE, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JSET_X: if (0x3 & 0xffffffff) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 0xffffffff),
+ BPF_JMP_REG(BPF_JNE, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JA: Jump, gap, jump, ...",
+ { },
+ CLASSIC | FLAG_NO_DATA,
+ { },
+ { { 0, 0xababcbac } },
+ .fill_helper = bpf_fill_ja,
+ },
+ { /* Mainly checking JIT here. */
+ "BPF_MAXINSNS: Maximum possible literals",
+ { },
+ CLASSIC | FLAG_NO_DATA,
+ { },
+ { { 0, 0xffffffff } },
+ .fill_helper = bpf_fill_maxinsns1,
+ },
+ { /* Mainly checking JIT here. */
+ "BPF_MAXINSNS: Single literal",
+ { },
+ CLASSIC | FLAG_NO_DATA,
+ { },
+ { { 0, 0xfefefefe } },
+ .fill_helper = bpf_fill_maxinsns2,
+ },
+ { /* Mainly checking JIT here. */
+ "BPF_MAXINSNS: Run/add until end",
+ { },
+ CLASSIC | FLAG_NO_DATA,
+ { },
+ { { 0, 0x947bf368 } },
+ .fill_helper = bpf_fill_maxinsns3,
+ },
+ {
+ "BPF_MAXINSNS: Too many instructions",
+ { },
+ CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL,
+ { },
+ { },
+ .fill_helper = bpf_fill_maxinsns4,
+ },
+ { /* Mainly checking JIT here. */
+ "BPF_MAXINSNS: Very long jump",
+ { },
+ CLASSIC | FLAG_NO_DATA,
+ { },
+ { { 0, 0xabababab } },
+ .fill_helper = bpf_fill_maxinsns5,
+ },
+ { /* Mainly checking JIT here. */
+ "BPF_MAXINSNS: Ctx heavy transformations",
+ { },
+ CLASSIC,
+ { },
+ {
+ { 1, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) },
+ { 10, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) }
+ },
+ .fill_helper = bpf_fill_maxinsns6,
+ },
+ { /* Mainly checking JIT here. */
+ "BPF_MAXINSNS: Call heavy transformations",
+ { },
+ CLASSIC | FLAG_NO_DATA,
+ { },
+ { { 1, 0 }, { 10, 0 } },
+ .fill_helper = bpf_fill_maxinsns7,
+ },
+ { /* Mainly checking JIT here. */
+ "BPF_MAXINSNS: Jump heavy test",
+ { },
+ CLASSIC | FLAG_NO_DATA,
+ { },
+ { { 0, 0xffffffff } },
+ .fill_helper = bpf_fill_maxinsns8,
+ },
+ { /* Mainly checking JIT here. */
+ "BPF_MAXINSNS: Very long jump backwards",
+ { },
+ INTERNAL | FLAG_NO_DATA,
+ { },
+ { { 0, 0xcbababab } },
+ .fill_helper = bpf_fill_maxinsns9,
+ },
+ { /* Mainly checking JIT here. */
+ "BPF_MAXINSNS: Edge hopping nuthouse",
+ { },
+ INTERNAL | FLAG_NO_DATA,
+ { },
+ { { 0, 0xabababac } },
+ .fill_helper = bpf_fill_maxinsns10,
+ },
+ {
+ "BPF_MAXINSNS: Jump, gap, jump, ...",
+ { },
+ CLASSIC | FLAG_NO_DATA,
+ { },
+ { { 0, 0xababcbac } },
+ .fill_helper = bpf_fill_maxinsns11,
+ },
};
static struct net_device dev;
@@ -1858,10 +4445,15 @@ static void release_test_data(const struct bpf_test *test, void *data)
kfree_skb(data);
}
-static int probe_filter_length(struct sock_filter *fp)
+static int filter_length(int which)
{
- int len = 0;
+ struct sock_filter *fp;
+ int len;
+ if (tests[which].fill_helper)
+ return tests[which].u.ptr.len;
+
+ fp = tests[which].u.insns;
for (len = MAX_INSNS - 1; len > 0; --len)
if (fp[len].code != 0 || fp[len].k != 0)
break;
@@ -1869,16 +4461,25 @@ static int probe_filter_length(struct sock_filter *fp)
return len + 1;
}
+static void *filter_pointer(int which)
+{
+ if (tests[which].fill_helper)
+ return tests[which].u.ptr.insns;
+ else
+ return tests[which].u.insns;
+}
+
static struct bpf_prog *generate_filter(int which, int *err)
{
- struct bpf_prog *fp;
- struct sock_fprog_kern fprog;
- unsigned int flen = probe_filter_length(tests[which].u.insns);
__u8 test_type = tests[which].aux & TEST_TYPE_MASK;
+ unsigned int flen = filter_length(which);
+ void *fptr = filter_pointer(which);
+ struct sock_fprog_kern fprog;
+ struct bpf_prog *fp;
switch (test_type) {
case CLASSIC:
- fprog.filter = tests[which].u.insns;
+ fprog.filter = fptr;
fprog.len = flen;
*err = bpf_prog_create(&fp, &fprog);
@@ -1914,8 +4515,7 @@ static struct bpf_prog *generate_filter(int which, int *err)
}
fp->len = flen;
- memcpy(fp->insnsi, tests[which].u.insns_int,
- fp->len * sizeof(struct bpf_insn));
+ memcpy(fp->insnsi, fptr, fp->len * sizeof(struct bpf_insn));
bpf_prog_select_runtime(fp);
break;
@@ -1987,9 +4587,33 @@ static int run_one(const struct bpf_prog *fp, struct bpf_test *test)
return err_cnt;
}
+static __init int prepare_bpf_tests(void)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(tests); i++) {
+ if (tests[i].fill_helper &&
+ tests[i].fill_helper(&tests[i]) < 0)
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static __init void destroy_bpf_tests(void)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(tests); i++) {
+ if (tests[i].fill_helper)
+ kfree(tests[i].u.ptr.insns);
+ }
+}
+
static __init int test_bpf(void)
{
int i, err_cnt = 0, pass_cnt = 0;
+ int jit_cnt = 0, run_cnt = 0;
for (i = 0; i < ARRAY_SIZE(tests); i++) {
struct bpf_prog *fp;
@@ -2006,6 +4630,13 @@ static __init int test_bpf(void)
return err;
}
+
+ pr_cont("jited:%u ", fp->jited);
+
+ run_cnt++;
+ if (fp->jited)
+ jit_cnt++;
+
err = run_one(fp, &tests[i]);
release_filter(fp, i);
@@ -2018,13 +4649,24 @@ static __init int test_bpf(void)
}
}
- pr_info("Summary: %d PASSED, %d FAILED\n", pass_cnt, err_cnt);
+ pr_info("Summary: %d PASSED, %d FAILED, [%d/%d JIT'ed]\n",
+ pass_cnt, err_cnt, jit_cnt, run_cnt);
+
return err_cnt ? -EINVAL : 0;
}
static int __init test_bpf_init(void)
{
- return test_bpf();
+ int ret;
+
+ ret = prepare_bpf_tests();
+ if (ret < 0)
+ return ret;
+
+ ret = test_bpf();
+
+ destroy_bpf_tests();
+ return ret;
}
static void __exit test_bpf_exit(void)
diff --git a/lib/test_rhashtable.c b/lib/test_rhashtable.c
index b2957540d3c7..c90777eae1f8 100644
--- a/lib/test_rhashtable.c
+++ b/lib/test_rhashtable.c
@@ -1,14 +1,9 @@
/*
* Resizable, Scalable, Concurrent Hash Table
*
- * Copyright (c) 2014 Thomas Graf <tgraf@suug.ch>
+ * Copyright (c) 2014-2015 Thomas Graf <tgraf@suug.ch>
* Copyright (c) 2008-2014 Patrick McHardy <kaber@trash.net>
*
- * Based on the following paper:
- * https://www.usenix.org/legacy/event/atc11/tech/final_files/Triplett.pdf
- *
- * Code partially derived from nft_hash
- *
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
@@ -26,20 +21,37 @@
#include <linux/rhashtable.h>
#include <linux/slab.h>
+#define MAX_ENTRIES 1000000
+#define TEST_INSERT_FAIL INT_MAX
+
+static int entries = 50000;
+module_param(entries, int, 0);
+MODULE_PARM_DESC(entries, "Number of entries to add (default: 50000)");
+
+static int runs = 4;
+module_param(runs, int, 0);
+MODULE_PARM_DESC(runs, "Number of test runs per variant (default: 4)");
+
+static int max_size = 65536;
+module_param(max_size, int, 0);
+MODULE_PARM_DESC(runs, "Maximum table size (default: 65536)");
-#define TEST_HT_SIZE 8
-#define TEST_ENTRIES 2048
-#define TEST_PTR ((void *) 0xdeadbeef)
-#define TEST_NEXPANDS 4
+static bool shrinking = false;
+module_param(shrinking, bool, 0);
+MODULE_PARM_DESC(shrinking, "Enable automatic shrinking (default: off)");
+
+static int size = 8;
+module_param(size, int, 0);
+MODULE_PARM_DESC(size, "Initial size hint of table (default: 8)");
struct test_obj {
- void *ptr;
int value;
struct rhash_head node;
};
-static const struct rhashtable_params test_rht_params = {
- .nelem_hint = TEST_HT_SIZE,
+static struct test_obj array[MAX_ENTRIES];
+
+static struct rhashtable_params test_rht_params = {
.head_offset = offsetof(struct test_obj, node),
.key_offset = offsetof(struct test_obj, value),
.key_len = sizeof(int),
@@ -51,11 +63,14 @@ static int __init test_rht_lookup(struct rhashtable *ht)
{
unsigned int i;
- for (i = 0; i < TEST_ENTRIES * 2; i++) {
+ for (i = 0; i < entries * 2; i++) {
struct test_obj *obj;
bool expected = !(i % 2);
u32 key = i;
+ if (array[i / 2].value == TEST_INSERT_FAIL)
+ expected = false;
+
obj = rhashtable_lookup_fast(ht, &key, test_rht_params);
if (expected && !obj) {
@@ -66,9 +81,9 @@ static int __init test_rht_lookup(struct rhashtable *ht)
key);
return -EEXIST;
} else if (expected && obj) {
- if (obj->ptr != TEST_PTR || obj->value != i) {
- pr_warn("Test failed: Lookup value mismatch %p!=%p, %u!=%u\n",
- obj->ptr, TEST_PTR, obj->value, i);
+ if (obj->value != i) {
+ pr_warn("Test failed: Lookup value mismatch %u!=%u\n",
+ obj->value, i);
return -EINVAL;
}
}
@@ -77,129 +92,147 @@ static int __init test_rht_lookup(struct rhashtable *ht)
return 0;
}
-static void test_bucket_stats(struct rhashtable *ht, bool quiet)
+static void test_bucket_stats(struct rhashtable *ht)
{
- unsigned int cnt, rcu_cnt, i, total = 0;
+ unsigned int err, total = 0, chain_len = 0;
+ struct rhashtable_iter hti;
struct rhash_head *pos;
- struct test_obj *obj;
- struct bucket_table *tbl;
- tbl = rht_dereference_rcu(ht->tbl, ht);
- for (i = 0; i < tbl->size; i++) {
- rcu_cnt = cnt = 0;
+ err = rhashtable_walk_init(ht, &hti);
+ if (err) {
+ pr_warn("Test failed: allocation error");
+ return;
+ }
- if (!quiet)
- pr_info(" [%#4x/%u]", i, tbl->size);
+ err = rhashtable_walk_start(&hti);
+ if (err && err != -EAGAIN) {
+ pr_warn("Test failed: iterator failed: %d\n", err);
+ return;
+ }
- rht_for_each_entry_rcu(obj, pos, tbl, i, node) {
- cnt++;
- total++;
- if (!quiet)
- pr_cont(" [%p],", obj);
+ while ((pos = rhashtable_walk_next(&hti))) {
+ if (PTR_ERR(pos) == -EAGAIN) {
+ pr_info("Info: encountered resize\n");
+ chain_len++;
+ continue;
+ } else if (IS_ERR(pos)) {
+ pr_warn("Test failed: rhashtable_walk_next() error: %ld\n",
+ PTR_ERR(pos));
+ break;
}
- rht_for_each_entry_rcu(obj, pos, tbl, i, node)
- rcu_cnt++;
-
- if (rcu_cnt != cnt)
- pr_warn("Test failed: Chain count mismach %d != %d",
- cnt, rcu_cnt);
-
- if (!quiet)
- pr_cont("\n [%#x] first element: %p, chain length: %u\n",
- i, tbl->buckets[i], cnt);
+ total++;
}
- pr_info(" Traversal complete: counted=%u, nelems=%u, entries=%d\n",
- total, atomic_read(&ht->nelems), TEST_ENTRIES);
+ rhashtable_walk_stop(&hti);
+ rhashtable_walk_exit(&hti);
+
+ pr_info(" Traversal complete: counted=%u, nelems=%u, entries=%d, table-jumps=%u\n",
+ total, atomic_read(&ht->nelems), entries, chain_len);
- if (total != atomic_read(&ht->nelems) || total != TEST_ENTRIES)
+ if (total != atomic_read(&ht->nelems) || total != entries)
pr_warn("Test failed: Total count mismatch ^^^");
}
-static int __init test_rhashtable(struct rhashtable *ht)
+static s64 __init test_rhashtable(struct rhashtable *ht)
{
- struct bucket_table *tbl;
struct test_obj *obj;
- struct rhash_head *pos, *next;
int err;
- unsigned int i;
+ unsigned int i, insert_fails = 0;
+ s64 start, end;
/*
* Insertion Test:
- * Insert TEST_ENTRIES into table with all keys even numbers
+ * Insert entries into table with all keys even numbers
*/
- pr_info(" Adding %d keys\n", TEST_ENTRIES);
- for (i = 0; i < TEST_ENTRIES; i++) {
- struct test_obj *obj;
-
- obj = kzalloc(sizeof(*obj), GFP_KERNEL);
- if (!obj) {
- err = -ENOMEM;
- goto error;
- }
+ pr_info(" Adding %d keys\n", entries);
+ start = ktime_get_ns();
+ for (i = 0; i < entries; i++) {
+ struct test_obj *obj = &array[i];
- obj->ptr = TEST_PTR;
obj->value = i * 2;
err = rhashtable_insert_fast(ht, &obj->node, test_rht_params);
- if (err) {
- kfree(obj);
- goto error;
+ if (err == -ENOMEM || err == -EBUSY) {
+ /* Mark failed inserts but continue */
+ obj->value = TEST_INSERT_FAIL;
+ insert_fails++;
+ } else if (err) {
+ return err;
}
}
+ if (insert_fails)
+ pr_info(" %u insertions failed due to memory pressure\n",
+ insert_fails);
+
+ test_bucket_stats(ht);
rcu_read_lock();
- test_bucket_stats(ht, true);
test_rht_lookup(ht);
rcu_read_unlock();
- rcu_read_lock();
- test_bucket_stats(ht, true);
- rcu_read_unlock();
+ test_bucket_stats(ht);
- pr_info(" Deleting %d keys\n", TEST_ENTRIES);
- for (i = 0; i < TEST_ENTRIES; i++) {
+ pr_info(" Deleting %d keys\n", entries);
+ for (i = 0; i < entries; i++) {
u32 key = i * 2;
- obj = rhashtable_lookup_fast(ht, &key, test_rht_params);
- BUG_ON(!obj);
+ if (array[i].value != TEST_INSERT_FAIL) {
+ obj = rhashtable_lookup_fast(ht, &key, test_rht_params);
+ BUG_ON(!obj);
- rhashtable_remove_fast(ht, &obj->node, test_rht_params);
- kfree(obj);
+ rhashtable_remove_fast(ht, &obj->node, test_rht_params);
+ }
}
- return 0;
-
-error:
- tbl = rht_dereference_rcu(ht->tbl, ht);
- for (i = 0; i < tbl->size; i++)
- rht_for_each_entry_safe(obj, pos, next, tbl, i, node)
- kfree(obj);
+ end = ktime_get_ns();
+ pr_info(" Duration of test: %lld ns\n", end - start);
- return err;
+ return end - start;
}
static struct rhashtable ht;
static int __init test_rht_init(void)
{
- int err;
+ int i, err;
+ u64 total_time = 0;
- pr_info("Running resizable hashtable tests...\n");
+ entries = min(entries, MAX_ENTRIES);
- err = rhashtable_init(&ht, &test_rht_params);
- if (err < 0) {
- pr_warn("Test failed: Unable to initialize hashtable: %d\n",
- err);
- return err;
- }
+ test_rht_params.automatic_shrinking = shrinking;
+ test_rht_params.max_size = max_size;
+ test_rht_params.nelem_hint = size;
- err = test_rhashtable(&ht);
+ pr_info("Running rhashtable test nelem=%d, max_size=%d, shrinking=%d\n",
+ size, max_size, shrinking);
- rhashtable_destroy(&ht);
+ for (i = 0; i < runs; i++) {
+ s64 time;
- return err;
+ pr_info("Test %02d:\n", i);
+ memset(&array, 0, sizeof(array));
+ err = rhashtable_init(&ht, &test_rht_params);
+ if (err < 0) {
+ pr_warn("Test failed: Unable to initialize hashtable: %d\n",
+ err);
+ continue;
+ }
+
+ time = test_rhashtable(&ht);
+ rhashtable_destroy(&ht);
+ if (time < 0) {
+ pr_warn("Test failed: return code %lld\n", time);
+ return -EINVAL;
+ }
+
+ total_time += time;
+ }
+
+ do_div(total_time, runs);
+ pr_info("Average test time: %llu\n", total_time);
+
+ return 0;
}
static void __exit test_rht_exit(void)