3 # Copyright (C) 2017 Netronome Systems, Inc.
4 # Copyright (c) 2019 Mellanox Technologies. All rights reserved
6 # This software is licensed under the GNU General License Version 2,
7 # June 1991 as shown in the file COPYING in the top-level directory of this
10 # THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
11 # WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
12 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
13 # FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
14 # OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
15 # THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
17 from datetime import datetime
35 bpf_test_dir = os.path.dirname(os.path.realpath(__file__))
36 pp = pprint.PrettyPrinter()
37 devs = [] # devices we created for clean up
38 files = [] # files to be removed
39 netns = [] # net namespaces to be removed
41 def log_get_sec(level=0):
42 return "*" * (log_level + level)
44 def log_level_inc(add=1):
48 def log_level_dec(sub=1):
52 def log_level_set(level):
56 def log(header, data, level=None):
58 Output to an optional log.
65 if not isinstance(data, str):
66 data = pp.pformat(data)
69 logfile.write("\n" + log_get_sec() + " ")
71 if len(header) and len(data.strip()):
79 log("SKIP: " + msg, "", level=1)
86 tb = "".join(traceback.extract_stack().format())
88 log("FAIL: " + msg, tb, level=1)
96 def cmd(cmd, shell=True, include_stderr=False, background=False, fail=True):
98 Run a command in subprocess and return tuple of (retval, stdout);
99 optionally return stderr as well as third value.
101 proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE,
102 stderr=subprocess.PIPE)
104 msg = "%s START: %s" % (log_get_sec(1),
105 datetime.now().strftime("%H:%M:%S.%f"))
106 log("BKG " + proc.args, msg)
109 return cmd_result(proc, include_stderr=include_stderr, fail=fail)
111 def cmd_result(proc, include_stderr=False, fail=False):
112 stdout, stderr = proc.communicate()
113 stdout = stdout.decode("utf-8")
114 stderr = stderr.decode("utf-8")
118 stderr = "\n" + stderr
119 if stderr[-1] == "\n":
123 log("CMD " + proc.args,
124 "RETCODE: %d\n%s STDOUT:\n%s%s STDERR:%s\n%s END: %s" %
125 (proc.returncode, sec, stdout, sec, stderr,
126 sec, datetime.now().strftime("%H:%M:%S.%f")))
128 if proc.returncode != 0 and fail:
129 if len(stderr) > 0 and stderr[-1] == "\n":
131 raise Exception("Command failed: %s\n%s" % (proc.args, stderr))
134 return proc.returncode, stdout, stderr
136 return proc.returncode, stdout
139 cmd("rm -f %s" % (f))
143 def tool(name, args, flags, JSON=True, ns="", fail=True, include_stderr=False):
146 params += "%s " % (flags["json"])
149 ns = "ip netns exec %s " % (ns)
152 ret, stdout, stderr = cmd(ns + name + " " + params + args,
153 fail=fail, include_stderr=True)
155 ret, stdout = cmd(ns + name + " " + params + args,
156 fail=fail, include_stderr=False)
158 if JSON and len(stdout.strip()) != 0:
159 out = json.loads(stdout)
164 return ret, out, stderr
168 def bpftool(args, JSON=True, ns="", fail=True, include_stderr=False):
169 return tool("bpftool", args, {"json":"-p"}, JSON=JSON, ns=ns,
170 fail=fail, include_stderr=include_stderr)
172 def bpftool_prog_list(expected=None, ns=""):
173 _, progs = bpftool("prog show", JSON=True, ns=ns, fail=True)
174 # Remove the base progs
178 if expected is not None:
179 if len(progs) != expected:
180 fail(True, "%d BPF programs loaded, expected %d" %
181 (len(progs), expected))
184 def bpftool_map_list(expected=None, ns=""):
185 _, maps = bpftool("map show", JSON=True, ns=ns, fail=True)
186 # Remove the base maps
190 if expected is not None:
191 if len(maps) != expected:
192 fail(True, "%d BPF maps loaded, expected %d" %
193 (len(maps), expected))
196 def bpftool_prog_list_wait(expected=0, n_retry=20):
197 for i in range(n_retry):
198 nprogs = len(bpftool_prog_list())
199 if nprogs == expected:
202 raise Exception("Time out waiting for program counts to stabilize want %d, have %d" % (expected, nprogs))
204 def bpftool_map_list_wait(expected=0, n_retry=20):
205 for i in range(n_retry):
206 nmaps = len(bpftool_map_list())
207 if nmaps == expected:
210 raise Exception("Time out waiting for map counts to stabilize want %d, have %d" % (expected, nmaps))
212 def bpftool_prog_load(sample, file_name, maps=[], prog_type="xdp", dev=None,
213 fail=True, include_stderr=False):
214 args = "prog load %s %s" % (os.path.join(bpf_test_dir, sample), file_name)
215 if prog_type is not None:
216 args += " type " + prog_type
218 args += " dev " + dev
220 args += " map " + " map ".join(maps)
222 res = bpftool(args, fail=fail, include_stderr=include_stderr)
224 files.append(file_name)
227 def ip(args, force=False, JSON=True, ns="", fail=True, include_stderr=False):
229 args = "-force " + args
230 return tool("ip", args, {"json":"-j"}, JSON=JSON, ns=ns,
231 fail=fail, include_stderr=include_stderr)
233 def tc(args, JSON=True, ns="", fail=True, include_stderr=False):
234 return tool("tc", args, {"json":"-p"}, JSON=JSON, ns=ns,
235 fail=fail, include_stderr=include_stderr)
237 def ethtool(dev, opt, args, fail=True):
238 return cmd("ethtool %s %s %s" % (opt, dev["ifname"], args), fail=fail)
240 def bpf_obj(name, sec=".text", path=bpf_test_dir,):
241 return "obj %s sec %s" % (os.path.join(path, name), sec)
243 def bpf_pinned(name):
244 return "pinned %s" % (name)
246 def bpf_bytecode(bytecode):
247 return "bytecode \"%s\"" % (bytecode)
249 def mknetns(n_retry=10):
250 for i in range(n_retry):
251 name = ''.join([random.choice(string.ascii_letters) for i in range(8)])
252 ret, _ = ip("netns add %s" % (name), fail=False)
258 def int2str(fmt, val):
260 for b in struct.pack(fmt, val):
262 return " ".join(map(lambda x: str(x), ret))
267 inttab.append(int(i, 16))
268 ba = bytearray(inttab)
271 elif len(strtab) == 8:
274 raise Exception("String array of len %d can't be unpacked to an int" %
276 return struct.unpack(fmt, ba)[0]
280 Class for accessing DebugFS directories as a dictionary.
283 def __init__(self, path):
285 self._dict = self._debugfs_dir_read(path)
288 return len(self._dict.keys())
290 def __getitem__(self, key):
292 key = list(self._dict.keys())[key]
293 return self._dict[key]
295 def __setitem__(self, key, value):
296 log("DebugFS set %s = %s" % (key, value), "")
299 cmd("echo '%s' > %s/%s" % (value, self.path, key))
302 _, out = cmd('cat %s/%s' % (self.path, key))
303 self._dict[key] = out.strip()
305 def _debugfs_dir_read(self, path):
308 log("DebugFS state for %s" % (path), "")
311 _, out = cmd('ls ' + path)
312 for f in out.split():
316 p = os.path.join(path, f)
317 if not os.stat(p).st_mode & stat.S_IRUSR:
320 if os.path.isfile(p):
321 # We need to init trap_flow_action_cookie before read it
322 if f == "trap_flow_action_cookie":
323 cmd('echo deadbeef > %s/%s' % (path, f))
324 _, out = cmd('cat %s/%s' % (path, f))
326 elif os.path.isdir(p):
327 dfs[f] = DebugfsDir(p)
329 raise Exception("%s is neither file nor directory" % (p))
332 log("DebugFS state", dfs)
339 Class for netdevsim bus device and its attributes.
342 def ctrl_write(path, val):
343 fullpath = os.path.join("/sys/bus/netdevsim/", path)
345 with open(fullpath, "w") as f:
348 log("WRITE %s: %r" % (fullpath, val), -e.errno)
350 log("WRITE %s: %r" % (fullpath, val), 0)
352 def __init__(self, port_count=1):
356 self.ctrl_write("new_device", "%u %u" % (addr, port_count))
358 if e.errno == errno.ENOSPC:
365 # As probe of netdevsim device might happen from a workqueue,
366 # so wait here until all netdevs appear.
367 self.wait_for_netdevs(port_count)
369 ret, out = cmd("udevadm settle", fail=False)
371 raise Exception("udevadm settle failed")
372 ifnames = self.get_ifnames()
375 self.dfs_dir = "/sys/kernel/debug/netdevsim/netdevsim%u/" % addr
378 for port_index in range(port_count):
379 self.nsims.append(NetdevSim(self, port_index, ifnames[port_index]))
381 def get_ifnames(self):
383 listdir = os.listdir("/sys/bus/netdevsim/devices/netdevsim%u/net/" % self.addr)
384 for ifname in listdir:
385 ifnames.append(ifname)
389 def wait_for_netdevs(self, port_count):
391 timeout_start = time.time()
395 ifnames = self.get_ifnames()
396 except FileNotFoundError as e:
398 if len(ifnames) == port_count:
400 if time.time() < timeout_start + timeout:
402 raise Exception("netdevices did not appear within timeout")
404 def dfs_num_bound_progs(self):
405 path = os.path.join(self.dfs_dir, "bpf_bound_progs")
406 _, progs = cmd('ls %s' % (path))
407 return len(progs.split())
409 def dfs_get_bound_progs(self, expected):
410 progs = DebugfsDir(os.path.join(self.dfs_dir, "bpf_bound_progs"))
411 if expected is not None:
412 if len(progs) != expected:
413 fail(True, "%d BPF programs bound, expected %d" %
414 (len(progs), expected))
418 self.ctrl_write("del_device", "%u" % (self.addr, ))
421 def remove_nsim(self, nsim):
422 self.nsims.remove(nsim)
423 self.ctrl_write("devices/netdevsim%u/del_port" % (self.addr, ),
424 "%u" % (nsim.port_index, ))
428 Class for netdevsim netdevice and its attributes.
431 def __init__(self, nsimdev, port_index, ifname):
432 # In case udev renamed the netdev to according to new schema,
433 # check if the name matches the port_index.
434 nsimnamere = re.compile("eni\d+np(\d+)")
435 match = nsimnamere.match(ifname)
436 if match and int(match.groups()[0]) != port_index + 1:
437 raise Exception("netdevice name mismatches the expected one")
439 self.nsimdev = nsimdev
440 self.port_index = port_index
442 self.dfs_dir = "%s/ports/%u/" % (nsimdev.dfs_dir, port_index)
444 _, [self.dev] = ip("link show dev %s" % ifname)
446 def __getitem__(self, key):
450 self.nsimdev.remove_nsim(self)
452 def dfs_refresh(self):
453 self.dfs = DebugfsDir(self.dfs_dir)
456 def dfs_read(self, f):
457 path = os.path.join(self.dfs_dir, f)
458 _, data = cmd('cat %s' % (path))
461 def wait_for_flush(self, bound=0, total=0, n_retry=20):
462 for i in range(n_retry):
463 nbound = self.nsimdev.dfs_num_bound_progs()
464 nprogs = len(bpftool_prog_list())
465 if nbound == bound and nprogs == total:
468 raise Exception("Time out waiting for program counts to stabilize want %d/%d, have %d bound, %d loaded" % (bound, total, nbound, nprogs))
470 def set_ns(self, ns):
471 name = "1" if ns == "" else ns
472 ip("link set dev %s netns %s" % (self.dev["ifname"], name), ns=self.ns)
475 def set_mtu(self, mtu, fail=True):
476 return ip("link set dev %s mtu %d" % (self.dev["ifname"], mtu),
479 def set_xdp(self, bpf, mode, force=False, JSON=True, verbose=False,
480 fail=True, include_stderr=False):
483 return ip("link set dev %s xdp%s %s" % (self.dev["ifname"], mode, bpf),
484 force=force, JSON=JSON,
485 fail=fail, include_stderr=include_stderr)
487 def unset_xdp(self, mode, force=False, JSON=True,
488 fail=True, include_stderr=False):
489 return ip("link set dev %s xdp%s off" % (self.dev["ifname"], mode),
490 force=force, JSON=JSON,
491 fail=fail, include_stderr=include_stderr)
493 def ip_link_show(self, xdp):
494 _, link = ip("link show dev %s" % (self['ifname']))
496 raise Exception("Multiple objects on ip link show")
499 fail(xdp != "xdp" in link,
500 "XDP program not reporting in iplink (reported %s, expected %s)" %
501 ("xdp" in link, xdp))
504 def tc_add_ingress(self):
505 tc("qdisc add dev %s ingress" % (self['ifname']))
507 def tc_del_ingress(self):
508 tc("qdisc del dev %s ingress" % (self['ifname']))
510 def tc_flush_filters(self, bound=0, total=0):
511 self.tc_del_ingress()
512 self.tc_add_ingress()
513 self.wait_for_flush(bound=bound, total=total)
515 def tc_show_ingress(self, expected=None):
516 # No JSON support, oh well...
517 flags = ["skip_sw", "skip_hw", "in_hw"]
518 named = ["protocol", "pref", "chain", "handle", "id", "tag"]
520 args = "-s filter show dev %s ingress" % (self['ifname'])
521 _, out = tc(args, JSON=False)
524 lines = out.split('\n')
527 if "handle" not in words:
531 fltr[flag] = flag in words
534 idx = words.index(name)
535 fltr[name] = words[idx + 1]
540 if expected is not None:
541 fail(len(filters) != expected,
542 "%d ingress filters loaded, expected %d" %
543 (len(filters), expected))
546 def cls_filter_op(self, op, qdisc="ingress", prio=None, handle=None,
547 chain=None, cls="", params="",
548 fail=True, include_stderr=False):
551 spec += " prio %d" % (prio)
553 spec += " handle %s" % (handle)
554 if chain is not None:
555 spec += " chain %d" % (chain)
557 return tc("filter {op} dev {dev} {qdisc} {spec} {cls} {params}"\
558 .format(op=op, dev=self['ifname'], qdisc=qdisc, spec=spec,
559 cls=cls, params=params),
560 fail=fail, include_stderr=include_stderr)
562 def cls_bpf_add_filter(self, bpf, op="add", prio=None, handle=None,
563 chain=None, da=False, verbose=False,
564 skip_sw=False, skip_hw=False,
565 fail=True, include_stderr=False):
578 return self.cls_filter_op(op=op, prio=prio, handle=handle, cls=cls,
579 chain=chain, params=params,
580 fail=fail, include_stderr=include_stderr)
582 def set_ethtool_tc_offloads(self, enable, fail=True):
583 args = "hw-tc-offload %s" % ("on" if enable else "off")
584 return ethtool(self, "-K", args, fail=fail)
586 ################################################################################
588 global files, netns, devs
593 cmd("rm -f %s" % (f))
595 cmd("ip netns delete %s" % (ns))
599 def pin_prog(file_name, idx=0):
600 progs = bpftool_prog_list(expected=(idx + 1))
602 bpftool("prog pin id %d %s" % (prog["id"], file_name))
603 files.append(file_name)
605 return file_name, bpf_pinned(file_name)
607 def pin_map(file_name, idx=0, expected=1):
608 maps = bpftool_map_list(expected=expected)
610 bpftool("map pin id %d %s" % (m["id"], file_name))
611 files.append(file_name)
613 return file_name, bpf_pinned(file_name)
615 def check_dev_info_removed(prog_file=None, map_file=None):
616 bpftool_prog_list(expected=0)
617 ret, err = bpftool("prog show pin %s" % (prog_file), fail=False)
618 fail(ret == 0, "Showing prog with removed device did not fail")
619 fail(err["error"].find("No such device") == -1,
620 "Showing prog with removed device expected ENODEV, error is %s" %
623 bpftool_map_list(expected=0)
624 ret, err = bpftool("map show pin %s" % (map_file), fail=False)
625 fail(ret == 0, "Showing map with removed device did not fail")
626 fail(err["error"].find("No such device") == -1,
627 "Showing map with removed device expected ENODEV, error is %s" %
630 def check_dev_info(other_ns, ns, prog_file=None, map_file=None, removed=False):
631 progs = bpftool_prog_list(expected=1, ns=ns)
634 fail("dev" not in prog.keys(), "Device parameters not reported")
636 fail("ifindex" not in dev.keys(), "Device parameters not reported")
637 fail("ns_dev" not in dev.keys(), "Device parameters not reported")
638 fail("ns_inode" not in dev.keys(), "Device parameters not reported")
641 fail("ifname" not in dev.keys(), "Ifname not reported")
642 fail(dev["ifname"] != sim["ifname"],
643 "Ifname incorrect %s vs %s" % (dev["ifname"], sim["ifname"]))
645 fail("ifname" in dev.keys(), "Ifname is reported for other ns")
647 maps = bpftool_map_list(expected=2, ns=ns)
649 fail("dev" not in m.keys(), "Device parameters not reported")
650 fail(dev != m["dev"], "Map's device different than program's")
652 def check_extack(output, reference, args):
655 lines = output.split("\n")
656 comp = len(lines) >= 2 and lines[1] == 'Error: ' + reference
657 fail(not comp, "Missing or incorrect netlink extack message")
659 def check_extack_nsim(output, reference, args):
660 check_extack(output, "netdevsim: " + reference, args)
662 def check_no_extack(res, needle):
663 fail((res[1] + res[2]).count(needle) or (res[1] + res[2]).count("Warning:"),
664 "Found '%s' in command output, leaky extack?" % (needle))
666 def check_verifier_log(output, reference):
667 lines = output.split("\n")
668 for l in reversed(lines):
671 fail(True, "Missing or incorrect message from netdevsim in verifier log")
673 def check_multi_basic(two_xdps):
674 fail(two_xdps["mode"] != 4, "Bad mode reported with multiple programs")
675 fail("prog" in two_xdps, "Base program reported in multi program mode")
676 fail(len(two_xdps["attached"]) != 2,
677 "Wrong attached program count with two programs")
678 fail(two_xdps["attached"][0]["prog"]["id"] ==
679 two_xdps["attached"][1]["prog"]["id"],
680 "Offloaded and other programs have the same id")
682 def test_spurios_extack(sim, obj, skip_hw, needle):
683 res = sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_hw=skip_hw,
685 check_no_extack(res, needle)
686 res = sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1,
687 skip_hw=skip_hw, include_stderr=True)
688 check_no_extack(res, needle)
689 res = sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf",
691 check_no_extack(res, needle)
693 def test_multi_prog(simdev, sim, obj, modename, modeid):
694 start_test("Test multi-attachment XDP - %s + offload..." %
695 (modename or "default", ))
696 sim.set_xdp(obj, "offload")
697 xdp = sim.ip_link_show(xdp=True)["xdp"]
698 offloaded = sim.dfs_read("bpf_offloaded_id")
699 fail("prog" not in xdp, "Base program not reported in single program mode")
700 fail(len(xdp["attached"]) != 1,
701 "Wrong attached program count with one program")
703 sim.set_xdp(obj, modename)
704 two_xdps = sim.ip_link_show(xdp=True)["xdp"]
706 fail(xdp["attached"][0] not in two_xdps["attached"],
707 "Offload program not reported after other activated")
708 check_multi_basic(two_xdps)
710 offloaded2 = sim.dfs_read("bpf_offloaded_id")
711 fail(offloaded != offloaded2,
712 "Offload ID changed after loading other program")
714 start_test("Test multi-attachment XDP - replace...")
715 ret, _, err = sim.set_xdp(obj, "offload", fail=False, include_stderr=True)
716 fail(ret == 0, "Replaced one of programs without -force")
717 check_extack(err, "XDP program already attached.", args)
719 if modename == "" or modename == "drv":
720 othermode = "" if modename == "drv" else "drv"
721 start_test("Test multi-attachment XDP - detach...")
722 ret, _, err = sim.unset_xdp(othermode, force=True,
723 fail=False, include_stderr=True)
724 fail(ret == 0, "Removed program with a bad mode")
725 check_extack(err, "program loaded with different flags.", args)
727 sim.unset_xdp("offload")
728 xdp = sim.ip_link_show(xdp=True)["xdp"]
729 offloaded = sim.dfs_read("bpf_offloaded_id")
731 fail(xdp["mode"] != modeid, "Bad mode reported after multiple programs")
732 fail("prog" not in xdp,
733 "Base program not reported after multi program mode")
734 fail(xdp["attached"][0] not in two_xdps["attached"],
735 "Offload program not reported after other activated")
736 fail(len(xdp["attached"]) != 1,
737 "Wrong attached program count with remaining programs")
738 fail(offloaded != "0", "Offload ID reported with only other program left")
740 start_test("Test multi-attachment XDP - reattach...")
741 sim.set_xdp(obj, "offload")
742 two_xdps = sim.ip_link_show(xdp=True)["xdp"]
744 fail(xdp["attached"][0] not in two_xdps["attached"],
745 "Other program not reported after offload activated")
746 check_multi_basic(two_xdps)
748 start_test("Test multi-attachment XDP - device remove...")
751 simdev = NetdevSimDev()
753 sim.set_ethtool_tc_offloads(True)
757 parser = argparse.ArgumentParser()
758 parser.add_argument("--log", help="output verbose log to given file")
759 args = parser.parse_args()
761 logfile = open(args.log, 'w+')
762 logfile.write("# -*-Org-*-")
764 log("Prepare...", "", level=1)
768 skip(os.getuid() != 0, "test must be run as root")
771 ret, progs = bpftool("prog", fail=False)
772 skip(ret != 0, "bpftool not installed")
774 _, base_maps = bpftool("map")
777 ret, out = cmd("modprobe netdevsim", fail=False)
778 skip(ret != 0, "netdevsim module could not be loaded")
781 _, out = cmd("mount")
782 if out.find("/sys/kernel/debug type debugfs") == -1:
783 cmd("mount -t debugfs none /sys/kernel/debug")
785 # Check samples are compiled
786 samples = ["sample_ret0.o", "sample_map_ret0.o"]
788 ret, out = cmd("ls %s/%s" % (bpf_test_dir, s), fail=False)
789 skip(ret != 0, "sample %s/%s not found, please compile it" %
792 # Check if iproute2 is built with libmnl (needed by extack support)
793 _, _, err = cmd("tc qdisc delete dev lo handle 0",
794 fail=False, include_stderr=True)
795 if err.find("Error: Failed to find qdisc with specified handle.") == -1:
796 print("Warning: no extack message in iproute2 output, libmnl missing?")
797 log("Warning: no extack message in iproute2 output, libmnl missing?", "")
800 # Check if net namespaces seem to work
802 skip(ns is None, "Could not create a net namespace")
803 cmd("ip netns delete %s" % (ns))
807 obj = bpf_obj("sample_ret0.o")
808 bytecode = bpf_bytecode("1,6 0 0 4294967295,")
810 start_test("Test destruction of generic XDP...")
811 simdev = NetdevSimDev()
813 sim.set_xdp(obj, "generic")
815 bpftool_prog_list_wait(expected=0)
817 simdev = NetdevSimDev()
821 start_test("Test TC non-offloaded...")
822 ret, _ = sim.cls_bpf_add_filter(obj, skip_hw=True, fail=False)
823 fail(ret != 0, "Software TC filter did not load")
825 start_test("Test TC non-offloaded isn't getting bound...")
826 ret, _ = sim.cls_bpf_add_filter(obj, fail=False)
827 fail(ret != 0, "Software TC filter did not load")
828 simdev.dfs_get_bound_progs(expected=0)
830 sim.tc_flush_filters()
832 start_test("Test TC offloads are off by default...")
833 ret, _, err = sim.cls_bpf_add_filter(obj, skip_sw=True,
834 fail=False, include_stderr=True)
835 fail(ret == 0, "TC filter loaded without enabling TC offloads")
836 check_extack(err, "TC offload is disabled on net device.", args)
839 sim.set_ethtool_tc_offloads(True)
840 sim.dfs["bpf_tc_non_bound_accept"] = "Y"
842 start_test("Test TC offload by default...")
843 ret, _ = sim.cls_bpf_add_filter(obj, fail=False)
844 fail(ret != 0, "Software TC filter did not load")
845 simdev.dfs_get_bound_progs(expected=0)
846 ingress = sim.tc_show_ingress(expected=1)
848 fail(not fltr["in_hw"], "Filter not offloaded by default")
850 sim.tc_flush_filters()
852 start_test("Test TC cBPF bytcode tries offload by default...")
853 ret, _ = sim.cls_bpf_add_filter(bytecode, fail=False)
854 fail(ret != 0, "Software TC filter did not load")
855 simdev.dfs_get_bound_progs(expected=0)
856 ingress = sim.tc_show_ingress(expected=1)
858 fail(not fltr["in_hw"], "Bytecode not offloaded by default")
860 sim.tc_flush_filters()
861 sim.dfs["bpf_tc_non_bound_accept"] = "N"
863 start_test("Test TC cBPF unbound bytecode doesn't offload...")
864 ret, _, err = sim.cls_bpf_add_filter(bytecode, skip_sw=True,
865 fail=False, include_stderr=True)
866 fail(ret == 0, "TC bytecode loaded for offload")
867 check_extack_nsim(err, "netdevsim configured to reject unbound programs.",
871 start_test("Test non-0 chain offload...")
872 ret, _, err = sim.cls_bpf_add_filter(obj, chain=1, prio=1, handle=1,
874 fail=False, include_stderr=True)
875 fail(ret == 0, "Offloaded a filter to chain other than 0")
876 check_extack(err, "Driver supports only offload of chain 0.", args)
877 sim.tc_flush_filters()
879 start_test("Test TC replace...")
880 sim.cls_bpf_add_filter(obj, prio=1, handle=1)
881 sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1)
882 sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
884 sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_sw=True)
885 sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1, skip_sw=True)
886 sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
888 sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_hw=True)
889 sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1, skip_hw=True)
890 sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
892 start_test("Test TC replace bad flags...")
895 ret, _ = sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1,
896 skip_sw=(j == 1), skip_hw=(j == 2),
898 fail(bool(ret) != bool(j),
899 "Software TC incorrect load in replace test, iteration %d" %
901 sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
903 start_test("Test spurious extack from the driver...")
904 test_spurios_extack(sim, obj, False, "netdevsim")
905 test_spurios_extack(sim, obj, True, "netdevsim")
907 sim.set_ethtool_tc_offloads(False)
909 test_spurios_extack(sim, obj, False, "TC offload is disabled")
910 test_spurios_extack(sim, obj, True, "TC offload is disabled")
912 sim.set_ethtool_tc_offloads(True)
914 sim.tc_flush_filters()
916 start_test("Test TC offloads work...")
917 ret, _, err = sim.cls_bpf_add_filter(obj, verbose=True, skip_sw=True,
918 fail=False, include_stderr=True)
919 fail(ret != 0, "TC filter did not load with TC offloads enabled")
920 check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
922 start_test("Test TC offload basics...")
923 dfs = simdev.dfs_get_bound_progs(expected=1)
924 progs = bpftool_prog_list(expected=1)
925 ingress = sim.tc_show_ingress(expected=1)
930 fail(fltr["skip_hw"], "TC does reports 'skip_hw' on offloaded filter")
931 fail(not fltr["in_hw"], "TC does not report 'in_hw' for offloaded filter")
932 fail(not fltr["skip_sw"], "TC does not report 'skip_sw' back")
934 start_test("Test TC offload is device-bound...")
935 fail(str(prog["id"]) != fltr["id"], "Program IDs don't match")
936 fail(prog["tag"] != fltr["tag"], "Program tags don't match")
937 fail(fltr["id"] != dprog["id"], "Program IDs don't match")
938 fail(dprog["state"] != "xlated", "Offloaded program state not translated")
939 fail(dprog["loaded"] != "Y", "Offloaded program is not loaded")
941 start_test("Test disabling TC offloads is rejected while filters installed...")
942 ret, _ = sim.set_ethtool_tc_offloads(False, fail=False)
943 fail(ret == 0, "Driver should refuse to disable TC offloads with filters installed...")
945 start_test("Test qdisc removal frees things...")
946 sim.tc_flush_filters()
947 sim.tc_show_ingress(expected=0)
949 start_test("Test disabling TC offloads is OK without filters...")
950 ret, _ = sim.set_ethtool_tc_offloads(False, fail=False)
952 "Driver refused to disable TC offloads without filters installed...")
954 sim.set_ethtool_tc_offloads(True)
956 start_test("Test destroying device gets rid of TC filters...")
957 sim.cls_bpf_add_filter(obj, skip_sw=True)
959 bpftool_prog_list_wait(expected=0)
961 simdev = NetdevSimDev()
963 sim.set_ethtool_tc_offloads(True)
965 start_test("Test destroying device gets rid of XDP...")
966 sim.set_xdp(obj, "offload")
968 bpftool_prog_list_wait(expected=0)
970 simdev = NetdevSimDev()
972 sim.set_ethtool_tc_offloads(True)
974 start_test("Test XDP prog reporting...")
975 sim.set_xdp(obj, "drv")
976 ipl = sim.ip_link_show(xdp=True)
977 progs = bpftool_prog_list(expected=1)
978 fail(ipl["xdp"]["prog"]["id"] != progs[0]["id"],
979 "Loaded program has wrong ID")
981 start_test("Test XDP prog replace without force...")
982 ret, _ = sim.set_xdp(obj, "drv", fail=False)
983 fail(ret == 0, "Replaced XDP program without -force")
984 sim.wait_for_flush(total=1)
986 start_test("Test XDP prog replace with force...")
987 ret, _ = sim.set_xdp(obj, "drv", force=True, fail=False)
988 fail(ret != 0, "Could not replace XDP program with -force")
989 bpftool_prog_list_wait(expected=1)
990 ipl = sim.ip_link_show(xdp=True)
991 progs = bpftool_prog_list(expected=1)
992 fail(ipl["xdp"]["prog"]["id"] != progs[0]["id"],
993 "Loaded program has wrong ID")
994 fail("dev" in progs[0].keys(),
995 "Device parameters reported for non-offloaded program")
997 start_test("Test XDP prog replace with bad flags...")
998 ret, _, err = sim.set_xdp(obj, "generic", force=True,
999 fail=False, include_stderr=True)
1000 fail(ret == 0, "Replaced XDP program with a program in different mode")
1002 "native and generic XDP can't be active at the same time.",
1004 ret, _, err = sim.set_xdp(obj, "", force=True,
1005 fail=False, include_stderr=True)
1006 fail(ret == 0, "Replaced XDP program with a program in different mode")
1007 check_extack(err, "program loaded with different flags.", args)
1009 start_test("Test XDP prog remove with bad flags...")
1010 ret, _, err = sim.unset_xdp("", force=True,
1011 fail=False, include_stderr=True)
1012 fail(ret == 0, "Removed program with a bad mode")
1013 check_extack(err, "program loaded with different flags.", args)
1015 start_test("Test MTU restrictions...")
1016 ret, _ = sim.set_mtu(9000, fail=False)
1018 "Driver should refuse to increase MTU to 9000 with XDP loaded...")
1019 sim.unset_xdp("drv")
1020 bpftool_prog_list_wait(expected=0)
1022 ret, _, err = sim.set_xdp(obj, "drv", fail=False, include_stderr=True)
1023 fail(ret == 0, "Driver should refuse to load program with MTU of 9000...")
1024 check_extack_nsim(err, "MTU too large w/ XDP enabled.", args)
1027 sim.wait_for_flush()
1028 start_test("Test non-offload XDP attaching to HW...")
1029 bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/nooffload")
1030 nooffload = bpf_pinned("/sys/fs/bpf/nooffload")
1031 ret, _, err = sim.set_xdp(nooffload, "offload",
1032 fail=False, include_stderr=True)
1033 fail(ret == 0, "attached non-offloaded XDP program to HW")
1034 check_extack_nsim(err, "xdpoffload of non-bound program.", args)
1035 rm("/sys/fs/bpf/nooffload")
1037 start_test("Test offload XDP attaching to drv...")
1038 bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/offload",
1040 offload = bpf_pinned("/sys/fs/bpf/offload")
1041 ret, _, err = sim.set_xdp(offload, "drv", fail=False, include_stderr=True)
1042 fail(ret == 0, "attached offloaded XDP program to drv")
1043 check_extack(err, "using device-bound program without HW_MODE flag is not supported.", args)
1044 rm("/sys/fs/bpf/offload")
1045 sim.wait_for_flush()
1047 start_test("Test XDP offload...")
1048 _, _, err = sim.set_xdp(obj, "offload", verbose=True, include_stderr=True)
1049 ipl = sim.ip_link_show(xdp=True)
1050 link_xdp = ipl["xdp"]["prog"]
1051 progs = bpftool_prog_list(expected=1)
1053 fail(link_xdp["id"] != prog["id"], "Loaded program has wrong ID")
1054 check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
1056 start_test("Test XDP offload is device bound...")
1057 dfs = simdev.dfs_get_bound_progs(expected=1)
1060 fail(prog["id"] != link_xdp["id"], "Program IDs don't match")
1061 fail(prog["tag"] != link_xdp["tag"], "Program tags don't match")
1062 fail(str(link_xdp["id"]) != dprog["id"], "Program IDs don't match")
1063 fail(dprog["state"] != "xlated", "Offloaded program state not translated")
1064 fail(dprog["loaded"] != "Y", "Offloaded program is not loaded")
1066 start_test("Test removing XDP program many times...")
1067 sim.unset_xdp("offload")
1068 sim.unset_xdp("offload")
1069 sim.unset_xdp("drv")
1070 sim.unset_xdp("drv")
1073 bpftool_prog_list_wait(expected=0)
1075 start_test("Test attempt to use a program for a wrong device...")
1076 simdev2 = NetdevSimDev()
1077 sim2, = simdev2.nsims
1078 sim2.set_xdp(obj, "offload")
1079 pin_file, pinned = pin_prog("/sys/fs/bpf/tmp")
1081 ret, _, err = sim.set_xdp(pinned, "offload",
1082 fail=False, include_stderr=True)
1083 fail(ret == 0, "Pinned program loaded for a different device accepted")
1084 check_extack_nsim(err, "program bound to different dev.", args)
1086 ret, _, err = sim.set_xdp(pinned, "offload",
1087 fail=False, include_stderr=True)
1088 fail(ret == 0, "Pinned program loaded for a removed device accepted")
1089 check_extack_nsim(err, "xdpoffload of non-bound program.", args)
1091 bpftool_prog_list_wait(expected=0)
1093 simdev, sim = test_multi_prog(simdev, sim, obj, "", 1)
1094 simdev, sim = test_multi_prog(simdev, sim, obj, "drv", 1)
1095 simdev, sim = test_multi_prog(simdev, sim, obj, "generic", 2)
1097 start_test("Test mixing of TC and XDP...")
1098 sim.tc_add_ingress()
1099 sim.set_xdp(obj, "offload")
1100 ret, _, err = sim.cls_bpf_add_filter(obj, skip_sw=True,
1101 fail=False, include_stderr=True)
1102 fail(ret == 0, "Loading TC when XDP active should fail")
1103 check_extack_nsim(err, "driver and netdev offload states mismatch.", args)
1104 sim.unset_xdp("offload")
1105 sim.wait_for_flush()
1107 sim.cls_bpf_add_filter(obj, skip_sw=True)
1108 ret, _, err = sim.set_xdp(obj, "offload", fail=False, include_stderr=True)
1109 fail(ret == 0, "Loading XDP when TC active should fail")
1110 check_extack_nsim(err, "TC program is already loaded.", args)
1112 start_test("Test binding TC from pinned...")
1113 pin_file, pinned = pin_prog("/sys/fs/bpf/tmp")
1114 sim.tc_flush_filters(bound=1, total=1)
1115 sim.cls_bpf_add_filter(pinned, da=True, skip_sw=True)
1116 sim.tc_flush_filters(bound=1, total=1)
1118 start_test("Test binding XDP from pinned...")
1119 sim.set_xdp(obj, "offload")
1120 pin_file, pinned = pin_prog("/sys/fs/bpf/tmp2", idx=1)
1122 sim.set_xdp(pinned, "offload", force=True)
1123 sim.unset_xdp("offload")
1124 sim.set_xdp(pinned, "offload", force=True)
1125 sim.unset_xdp("offload")
1127 start_test("Test offload of wrong type fails...")
1128 ret, _ = sim.cls_bpf_add_filter(pinned, da=True, skip_sw=True, fail=False)
1129 fail(ret == 0, "Managed to attach XDP program to TC")
1131 start_test("Test asking for TC offload of two filters...")
1132 sim.cls_bpf_add_filter(obj, da=True, skip_sw=True)
1133 ret, _, err = sim.cls_bpf_add_filter(obj, da=True, skip_sw=True,
1134 fail=False, include_stderr=True)
1135 fail(ret == 0, "Managed to offload two TC filters at the same time")
1136 check_extack_nsim(err, "driver and netdev offload states mismatch.", args)
1138 sim.tc_flush_filters(bound=2, total=2)
1140 start_test("Test if netdev removal waits for translation...")
1142 sim.dfs["dev/bpf_bind_verifier_delay"] = delay_msec
1144 cmd_line = "tc filter add dev %s ingress bpf %s da skip_sw" % \
1145 (sim['ifname'], obj)
1146 tc_proc = cmd(cmd_line, background=True, fail=False)
1147 # Wait for the verifier to start
1148 while simdev.dfs_num_bound_progs() <= 2:
1152 ret, _ = cmd_result(tc_proc, fail=False)
1153 time_diff = end - start
1154 log("Time", "start:\t%s\nend:\t%s\ndiff:\t%s" % (start, end, time_diff))
1156 fail(ret == 0, "Managed to load TC filter on a unregistering device")
1157 delay_sec = delay_msec * 0.001
1158 fail(time_diff < delay_sec, "Removal process took %s, expected %s" %
1159 (time_diff, delay_sec))
1161 # Remove all pinned files and reinstantiate the netdev
1163 bpftool_prog_list_wait(expected=0)
1165 simdev = NetdevSimDev()
1167 map_obj = bpf_obj("sample_map_ret0.o")
1168 start_test("Test loading program with maps...")
1169 sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1171 start_test("Test bpftool bound info reporting (own ns)...")
1172 check_dev_info(False, "")
1174 start_test("Test bpftool bound info reporting (other ns)...")
1177 check_dev_info(True, "")
1179 start_test("Test bpftool bound info reporting (remote ns)...")
1180 check_dev_info(False, ns)
1182 start_test("Test bpftool bound info reporting (back to own ns)...")
1184 check_dev_info(False, "")
1186 prog_file, _ = pin_prog("/sys/fs/bpf/tmp_prog")
1187 map_file, _ = pin_map("/sys/fs/bpf/tmp_map", idx=1, expected=2)
1190 start_test("Test bpftool bound info reporting (removed dev)...")
1191 check_dev_info_removed(prog_file=prog_file, map_file=map_file)
1193 # Remove all pinned files and reinstantiate the netdev
1195 bpftool_prog_list_wait(expected=0)
1197 simdev = NetdevSimDev()
1200 start_test("Test map update (no flags)...")
1201 sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1202 maps = bpftool_map_list(expected=2)
1203 array = maps[0] if maps[0]["type"] == "array" else maps[1]
1204 htab = maps[0] if maps[0]["type"] == "hash" else maps[1]
1207 bpftool("map update id %d key %s value %s" %
1208 (m["id"], int2str("I", i), int2str("Q", i * 3)))
1211 ret, _ = bpftool("map update id %d key %s value %s" %
1212 (m["id"], int2str("I", 3), int2str("Q", 3 * 3)),
1214 fail(ret == 0, "added too many entries")
1216 start_test("Test map update (exists)...")
1219 bpftool("map update id %d key %s value %s exist" %
1220 (m["id"], int2str("I", i), int2str("Q", i * 3)))
1223 ret, err = bpftool("map update id %d key %s value %s exist" %
1224 (m["id"], int2str("I", 3), int2str("Q", 3 * 3)),
1226 fail(ret == 0, "updated non-existing key")
1227 fail(err["error"].find("No such file or directory") == -1,
1228 "expected ENOENT, error is '%s'" % (err["error"]))
1230 start_test("Test map update (noexist)...")
1233 ret, err = bpftool("map update id %d key %s value %s noexist" %
1234 (m["id"], int2str("I", i), int2str("Q", i * 3)),
1236 fail(ret == 0, "updated existing key")
1237 fail(err["error"].find("File exists") == -1,
1238 "expected EEXIST, error is '%s'" % (err["error"]))
1240 start_test("Test map dump...")
1242 _, entries = bpftool("map dump id %d" % (m["id"]))
1244 key = str2int(entries[i]["key"])
1245 fail(key != i, "expected key %d, got %d" % (key, i))
1246 val = str2int(entries[i]["value"])
1247 fail(val != i * 3, "expected value %d, got %d" % (val, i * 3))
1249 start_test("Test map getnext...")
1251 _, entry = bpftool("map getnext id %d" % (m["id"]))
1252 key = str2int(entry["next_key"])
1253 fail(key != 0, "next key %d, expected %d" % (key, 0))
1254 _, entry = bpftool("map getnext id %d key %s" %
1255 (m["id"], int2str("I", 0)))
1256 key = str2int(entry["next_key"])
1257 fail(key != 1, "next key %d, expected %d" % (key, 1))
1258 ret, err = bpftool("map getnext id %d key %s" %
1259 (m["id"], int2str("I", 1)), fail=False)
1260 fail(ret == 0, "got next key past the end of map")
1261 fail(err["error"].find("No such file or directory") == -1,
1262 "expected ENOENT, error is '%s'" % (err["error"]))
1264 start_test("Test map delete (htab)...")
1266 bpftool("map delete id %d key %s" % (htab["id"], int2str("I", i)))
1268 start_test("Test map delete (array)...")
1270 ret, err = bpftool("map delete id %d key %s" %
1271 (htab["id"], int2str("I", i)), fail=False)
1272 fail(ret == 0, "removed entry from an array")
1273 fail(err["error"].find("No such file or directory") == -1,
1274 "expected ENOENT, error is '%s'" % (err["error"]))
1276 start_test("Test map remove...")
1277 sim.unset_xdp("offload")
1278 bpftool_map_list_wait(expected=0)
1281 simdev = NetdevSimDev()
1283 sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1285 bpftool_map_list_wait(expected=0)
1287 start_test("Test map creation fail path...")
1288 simdev = NetdevSimDev()
1290 sim.dfs["bpf_map_accept"] = "N"
1291 ret, _ = sim.set_xdp(map_obj, "offload", JSON=False, fail=False)
1293 "netdevsim didn't refuse to create a map with offload disabled")
1297 start_test("Test multi-dev ASIC program reuse...")
1298 simdevA = NetdevSimDev()
1299 simA, = simdevA.nsims
1300 simdevB = NetdevSimDev(3)
1301 simB1, simB2, simB3 = simdevB.nsims
1302 sims = (simA, simB1, simB2, simB3)
1303 simB = (simB1, simB2, simB3)
1305 bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA",
1307 progA = bpf_pinned("/sys/fs/bpf/nsimA")
1308 bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB",
1309 dev=simB1['ifname'])
1310 progB = bpf_pinned("/sys/fs/bpf/nsimB")
1312 simA.set_xdp(progA, "offload", JSON=False)
1313 for d in simdevB.nsims:
1314 d.set_xdp(progB, "offload", JSON=False)
1316 start_test("Test multi-dev ASIC cross-dev replace...")
1317 ret, _ = simA.set_xdp(progB, "offload", force=True, JSON=False, fail=False)
1318 fail(ret == 0, "cross-ASIC program allowed")
1319 for d in simdevB.nsims:
1320 ret, _ = d.set_xdp(progA, "offload", force=True, JSON=False, fail=False)
1321 fail(ret == 0, "cross-ASIC program allowed")
1323 start_test("Test multi-dev ASIC cross-dev install...")
1325 d.unset_xdp("offload")
1327 ret, _, err = simA.set_xdp(progB, "offload", force=True, JSON=False,
1328 fail=False, include_stderr=True)
1329 fail(ret == 0, "cross-ASIC program allowed")
1330 check_extack_nsim(err, "program bound to different dev.", args)
1331 for d in simdevB.nsims:
1332 ret, _, err = d.set_xdp(progA, "offload", force=True, JSON=False,
1333 fail=False, include_stderr=True)
1334 fail(ret == 0, "cross-ASIC program allowed")
1335 check_extack_nsim(err, "program bound to different dev.", args)
1337 start_test("Test multi-dev ASIC cross-dev map reuse...")
1339 mapA = bpftool("prog show %s" % (progA))[1]["map_ids"][0]
1340 mapB = bpftool("prog show %s" % (progB))[1]["map_ids"][0]
1342 ret, _ = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",
1343 dev=simB3['ifname'],
1344 maps=["idx 0 id %d" % (mapB)],
1346 fail(ret != 0, "couldn't reuse a map on the same ASIC")
1347 rm("/sys/fs/bpf/nsimB_")
1349 ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA_",
1351 maps=["idx 0 id %d" % (mapB)],
1352 fail=False, include_stderr=True)
1353 fail(ret == 0, "could reuse a map on a different ASIC")
1354 fail(err.count("offload device mismatch between prog and map") == 0,
1355 "error message missing for cross-ASIC map")
1357 ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",
1358 dev=simB1['ifname'],
1359 maps=["idx 0 id %d" % (mapA)],
1360 fail=False, include_stderr=True)
1361 fail(ret == 0, "could reuse a map on a different ASIC")
1362 fail(err.count("offload device mismatch between prog and map") == 0,
1363 "error message missing for cross-ASIC map")
1365 start_test("Test multi-dev ASIC cross-dev destruction...")
1366 bpftool_prog_list_wait(expected=2)
1369 bpftool_prog_list_wait(expected=1)
1371 ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
1372 fail(ifnameB != simB1['ifname'], "program not bound to original device")
1374 bpftool_prog_list_wait(expected=1)
1376 start_test("Test multi-dev ASIC cross-dev destruction - move...")
1377 ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
1378 fail(ifnameB not in (simB2['ifname'], simB3['ifname']),
1379 "program not bound to remaining devices")
1382 ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
1383 fail(ifnameB != simB3['ifname'], "program not bound to remaining device")
1387 bpftool_prog_list_wait(expected=0)
1389 start_test("Test multi-dev ASIC cross-dev destruction - orphaned...")
1390 ret, out = bpftool("prog show %s" % (progB), fail=False)
1391 fail(ret == 0, "got information about orphaned program")
1392 fail("error" not in out, "no error reported for get info on orphaned")
1393 fail(out["error"] != "can't get prog info: No such device",
1394 "wrong error for get info on orphaned")
1396 print("%s: OK" % (os.path.basename(__file__)))
1399 log("Clean up...", "", level=1)