[sashal-stable:pending-5.10 73/105] arch/powerpc/net/bpf_jit_comp64.c:415:46: error: implicit declaration of function 'PPC_LI'; did you mean 'PPC_HI'?
11:33 p.m.
tree: https://git.kernel.org/pub/scm/linux/kernel/git/sashal/linux-stable.git
pending-5.10
head: 1878c5ce146b59e4633771bab4fe7a9ca14d3ae4
commit: 190e3298b9a147fa55d76e148a7d0b0d34929898 [73/105] powerpc/bpf: Fix BPF_MOD when
imm == 1
config: powerpc-allmodconfig (attached as .config)
compiler: powerpc64-linux-gcc (GCC) 11.2.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O
~/bin/make.cross
chmod +x ~/bin/make.cross
#
https://git.kernel.org/pub/scm/linux/kernel/git/sashal/linux-stable.git/c...
git remote add sashal-stable
https://git.kernel.org/pub/scm/linux/kernel/git/sashal/linux-stable.git
git fetch --no-tags sashal-stable pending-5.10
git checkout 190e3298b9a147fa55d76e148a7d0b0d34929898
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-11.2.0 make.cross ARCH=powerpc
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp(a)intel.com>
All errors (new ones prefixed by >>):
In file included from arch/powerpc/net/bpf_jit64.h:11,
from arch/powerpc/net/bpf_jit_comp64.c:19:
arch/powerpc/net/bpf_jit_comp64.c: In function 'bpf_jit_build_body':
> arch/powerpc/net/bpf_jit_comp64.c:415:46: error: implicit
declaration of function 'PPC_LI'; did you mean 'PPC_HI'?
[-Werror=implicit-function-declaration]
415 |
EMIT(PPC_LI(dst_reg, 0));
| ^~~~~~
arch/powerpc/net/bpf_jit.h:23:34: note: in definition of macro 'PLANT_INSTR'
23 | do { if (d) { (d)[idx] = instr; } idx++; } while (0)
| ^~~~~
arch/powerpc/net/bpf_jit_comp64.c:415:41: note: in expansion of macro 'EMIT'
415 | EMIT(PPC_LI(dst_reg, 0));
| ^~~~
cc1: some warnings being treated as errors
vim +415 arch/powerpc/net/bpf_jit_comp64.c
290
291 /* Assemble the body code between the prologue & epilogue */
292 static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
293 struct codegen_context *ctx,
294 u32 *addrs, bool extra_pass)
295 {
296 const struct bpf_insn *insn = fp->insnsi;
297 int flen = fp->len;
298 int i, ret;
299
300 /* Start of epilogue code - will only be valid 2nd pass onwards */
301 u32 exit_addr = addrs[flen];
302
303 for (i = 0; i < flen; i++) {
304 u32 code = insn[i].code;
305 u32 dst_reg = b2p[insn[i].dst_reg];
306 u32 src_reg = b2p[insn[i].src_reg];
307 s16 off = insn[i].off;
308 s32 imm = insn[i].imm;
309 bool func_addr_fixed;
310 u64 func_addr;
311 u64 imm64;
312 u32 true_cond;
313 u32 tmp_idx;
314
315 /*
316 * addrs[] maps a BPF bytecode address into a real offset from
317 * the start of the body code.
318 */
319 addrs[i] = ctx->idx * 4;
320
321 /*
322 * As an optimization, we note down which non-volatile registers
323 * are used so that we can only save/restore those in our
324 * prologue and epilogue. We do this here regardless of whether
325 * the actual BPF instruction uses src/dst registers or not
326 * (for instance, BPF_CALL does not use them). The expectation
327 * is that those instructions will have src_reg/dst_reg set to
328 * 0. Even otherwise, we just lose some prologue/epilogue
329 * optimization but everything else should work without
330 * any issues.
331 */
332 if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32)
333 bpf_set_seen_register(ctx, insn[i].dst_reg);
334 if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32)
335 bpf_set_seen_register(ctx, insn[i].src_reg);
336
337 switch (code) {
338 /*
339 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
340 */
341 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
342 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
343 EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
344 goto bpf_alu32_trunc;
345 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
346 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
347 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
348 goto bpf_alu32_trunc;
349 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
350 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
351 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
352 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
353 if (BPF_OP(code) == BPF_SUB)
354 imm = -imm;
355 if (imm) {
356 if (imm >= -32768 && imm < 32768)
357 EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
358 else {
359 PPC_LI32(b2p[TMP_REG_1], imm);
360 EMIT(PPC_RAW_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]));
361 }
362 }
363 goto bpf_alu32_trunc;
364 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
365 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
366 if (BPF_CLASS(code) == BPF_ALU)
367 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
368 else
369 EMIT(PPC_RAW_MULD(dst_reg, dst_reg, src_reg));
370 goto bpf_alu32_trunc;
371 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
372 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
373 if (imm >= -32768 && imm < 32768)
374 EMIT(PPC_RAW_MULI(dst_reg, dst_reg, IMM_L(imm)));
375 else {
376 PPC_LI32(b2p[TMP_REG_1], imm);
377 if (BPF_CLASS(code) == BPF_ALU)
378 EMIT(PPC_RAW_MULW(dst_reg, dst_reg,
379 b2p[TMP_REG_1]));
380 else
381 EMIT(PPC_RAW_MULD(dst_reg, dst_reg,
382 b2p[TMP_REG_1]));
383 }
384 goto bpf_alu32_trunc;
385 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
386 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
387 if (BPF_OP(code) == BPF_MOD) {
388 EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg));
389 EMIT(PPC_RAW_MULW(b2p[TMP_REG_1], src_reg,
390 b2p[TMP_REG_1]));
391 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
392 } else
393 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
394 goto bpf_alu32_trunc;
395 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
396 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
397 if (BPF_OP(code) == BPF_MOD) {
398 EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_1], dst_reg, src_reg));
399 EMIT(PPC_RAW_MULD(b2p[TMP_REG_1], src_reg,
400 b2p[TMP_REG_1]));
401 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
402 } else
403 EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, src_reg));
404 break;
405 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
406 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
407 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
408 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
409 if (imm == 0)
410 return -EINVAL;
411 if (imm == 1) {
412 if (BPF_OP(code) == BPF_DIV) {
413 goto bpf_alu32_trunc;
414 } else {
415 EMIT(PPC_LI(dst_reg, 0));
416 break;
417 }
418 }
419
420 PPC_LI32(b2p[TMP_REG_1], imm);
421 switch (BPF_CLASS(code)) {
422 case BPF_ALU:
423 if (BPF_OP(code) == BPF_MOD) {
424 EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_2],
425 dst_reg,
426 b2p[TMP_REG_1]));
427 EMIT(PPC_RAW_MULW(b2p[TMP_REG_1],
428 b2p[TMP_REG_1],
429 b2p[TMP_REG_2]));
430 EMIT(PPC_RAW_SUB(dst_reg, dst_reg,
431 b2p[TMP_REG_1]));
432 } else
433 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg,
434 b2p[TMP_REG_1]));
435 break;
436 case BPF_ALU64:
437 if (BPF_OP(code) == BPF_MOD) {
438 EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_2],
439 dst_reg,
440 b2p[TMP_REG_1]));
441 EMIT(PPC_RAW_MULD(b2p[TMP_REG_1],
442 b2p[TMP_REG_1],
443 b2p[TMP_REG_2]));
444 EMIT(PPC_RAW_SUB(dst_reg, dst_reg,
445 b2p[TMP_REG_1]));
446 } else
447 EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg,
448 b2p[TMP_REG_1]));
449 break;
450 }
451 goto bpf_alu32_trunc;
452 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
453 case BPF_ALU64 | BPF_NEG: /* dst = -dst */
454 EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
455 goto bpf_alu32_trunc;
456
457 /*
458 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
459 */
460 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
461 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
462 EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
463 goto bpf_alu32_trunc;
464 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
465 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
466 if (!IMM_H(imm))
467 EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
468 else {
469 /* Sign-extended */
470 PPC_LI32(b2p[TMP_REG_1], imm);
471 EMIT(PPC_RAW_AND(dst_reg, dst_reg, b2p[TMP_REG_1]));
472 }
473 goto bpf_alu32_trunc;
474 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
475 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
476 EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
477 goto bpf_alu32_trunc;
478 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
479 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
480 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
481 /* Sign-extended */
482 PPC_LI32(b2p[TMP_REG_1], imm);
483 EMIT(PPC_RAW_OR(dst_reg, dst_reg, b2p[TMP_REG_1]));
484 } else {
485 if (IMM_L(imm))
486 EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
487 if (IMM_H(imm))
488 EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
489 }
490 goto bpf_alu32_trunc;
491 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
492 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
493 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
494 goto bpf_alu32_trunc;
495 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
496 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
497 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
498 /* Sign-extended */
499 PPC_LI32(b2p[TMP_REG_1], imm);
500 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]));
501 } else {
502 if (IMM_L(imm))
503 EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
504 if (IMM_H(imm))
505 EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
506 }
507 goto bpf_alu32_trunc;
508 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
509 /* slw clears top 32 bits */
510 EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
511 /* skip zero extension move, but set address map. */
512 if (insn_is_zext(&insn[i + 1]))
513 addrs[++i] = ctx->idx * 4;
514 break;
515 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
516 EMIT(PPC_RAW_SLD(dst_reg, dst_reg, src_reg));
517 break;
518 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */
519 /* with imm 0, we still need to clear top 32 bits */
520 EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
521 if (insn_is_zext(&insn[i + 1]))
522 addrs[++i] = ctx->idx * 4;
523 break;
524 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */
525 if (imm != 0)
526 EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, imm));
527 break;
528 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
529 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
530 if (insn_is_zext(&insn[i + 1]))
531 addrs[++i] = ctx->idx * 4;
532 break;
533 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
534 EMIT(PPC_RAW_SRD(dst_reg, dst_reg, src_reg));
535 break;
536 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
537 EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
538 if (insn_is_zext(&insn[i + 1]))
539 addrs[++i] = ctx->idx * 4;
540 break;
541 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
542 if (imm != 0)
543 EMIT(PPC_RAW_SRDI(dst_reg, dst_reg, imm));
544 break;
545 case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
546 EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg));
547 goto bpf_alu32_trunc;
548 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
549 EMIT(PPC_RAW_SRAD(dst_reg, dst_reg, src_reg));
550 break;
551 case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
552 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
553 goto bpf_alu32_trunc;
554 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
555 if (imm != 0)
556 EMIT(PPC_RAW_SRADI(dst_reg, dst_reg, imm));
557 break;
558
559 /*
560 * MOV
561 */
562 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
563 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
564 if (imm == 1) {
565 /* special mov32 for zext */
566 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
567 break;
568 }
569 EMIT(PPC_RAW_MR(dst_reg, src_reg));
570 goto bpf_alu32_trunc;
571 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
572 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
573 PPC_LI32(dst_reg, imm);
574 if (imm < 0)
575 goto bpf_alu32_trunc;
576 else if (insn_is_zext(&insn[i + 1]))
577 addrs[++i] = ctx->idx * 4;
578 break;
579
580 bpf_alu32_trunc:
581 /* Truncate to 32-bits */
582 if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext)
583 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
584 break;
585
586 /*
587 * BPF_FROM_BE/LE
588 */
589 case BPF_ALU | BPF_END | BPF_FROM_LE:
590 case BPF_ALU | BPF_END | BPF_FROM_BE:
591 #ifdef __BIG_ENDIAN__
592 if (BPF_SRC(code) == BPF_FROM_BE)
593 goto emit_clear;
594 #else /* !__BIG_ENDIAN__ */
595 if (BPF_SRC(code) == BPF_FROM_LE)
596 goto emit_clear;
597 #endif
598 switch (imm) {
599 case 16:
600 /* Rotate 8 bits left & mask with 0x0000ff00 */
601 EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23));
602 /* Rotate 8 bits right & insert LSB to reg */
603 EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31));
604 /* Move result back to dst_reg */
605 EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1]));
606 break;
607 case 32:
608 /*
609 * Rotate word left by 8 bits:
610 * 2 bytes are already in their final position
611 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
612 */
613 EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31));
614 /* Rotate 24 bits and insert byte 1 */
615 EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7));
616 /* Rotate 24 bits and insert byte 3 */
617 EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23));
618 EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1]));
619 break;
620 case 64:
621 /*
622 * Way easier and faster(?) to store the value
623 * into stack and then use ldbrx
624 *
625 * ctx->seen will be reliable in pass2, but
626 * the instructions generated will remain the
627 * same across all passes
628 */
629 PPC_BPF_STL(dst_reg, 1, bpf_jit_stack_local(ctx));
630 EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx)));
631 EMIT(PPC_RAW_LDBRX(dst_reg, 0, b2p[TMP_REG_1]));
632 break;
633 }
634 break;
635
636 emit_clear:
637 switch (imm) {
638 case 16:
639 /* zero-extend 16 bits into 64 bits */
640 EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 48));
641 if (insn_is_zext(&insn[i + 1]))
642 addrs[++i] = ctx->idx * 4;
643 break;
644 case 32:
645 if (!fp->aux->verifier_zext)
646 /* zero-extend 32 bits into 64 bits */
647 EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 32));
648 break;
649 case 64:
650 /* nop */
651 break;
652 }
653 break;
654
655 /*
656 * BPF_ST NOSPEC (speculation barrier)
657 */
658 case BPF_ST | BPF_NOSPEC:
659 break;
660
661 /*
662 * BPF_ST(X)
663 */
664 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
665 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
666 if (BPF_CLASS(code) == BPF_ST) {
667 EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
668 src_reg = b2p[TMP_REG_1];
669 }
670 EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
671 break;
672 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
673 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
674 if (BPF_CLASS(code) == BPF_ST) {
675 EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
676 src_reg = b2p[TMP_REG_1];
677 }
678 EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
679 break;
680 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
681 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
682 if (BPF_CLASS(code) == BPF_ST) {
683 PPC_LI32(b2p[TMP_REG_1], imm);
684 src_reg = b2p[TMP_REG_1];
685 }
686 EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
687 break;
688 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
689 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
690 if (BPF_CLASS(code) == BPF_ST) {
691 PPC_LI32(b2p[TMP_REG_1], imm);
692 src_reg = b2p[TMP_REG_1];
693 }
694 PPC_BPF_STL(src_reg, dst_reg, off);
695 break;
696
697 /*
698 * BPF_STX XADD (atomic_add)
699 */
700 /* *(u32 *)(dst + off) += src */
701 case BPF_STX | BPF_XADD | BPF_W:
702 /* Get EA into TMP_REG_1 */
703 EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off));
704 tmp_idx = ctx->idx * 4;
705 /* load value from memory into TMP_REG_2 */
706 EMIT(PPC_RAW_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0));
707 /* add value from src_reg into this */
708 EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg));
709 /* store result back */
710 EMIT(PPC_RAW_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]));
711 /* we're done if this succeeded */
712 PPC_BCC_SHORT(COND_NE, tmp_idx);
713 break;
714 /* *(u64 *)(dst + off) += src */
715 case BPF_STX | BPF_XADD | BPF_DW:
716 EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off));
717 tmp_idx = ctx->idx * 4;
718 EMIT(PPC_RAW_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0));
719 EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg));
720 EMIT(PPC_RAW_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]));
721 PPC_BCC_SHORT(COND_NE, tmp_idx);
722 break;
723
724 /*
725 * BPF_LDX
726 */
727 /* dst = *(u8 *)(ul) (src + off) */
728 case BPF_LDX | BPF_MEM | BPF_B:
729 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
730 if (insn_is_zext(&insn[i + 1]))
731 addrs[++i] = ctx->idx * 4;
732 break;
733 /* dst = *(u16 *)(ul) (src + off) */
734 case BPF_LDX | BPF_MEM | BPF_H:
735 EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
736 if (insn_is_zext(&insn[i + 1]))
737 addrs[++i] = ctx->idx * 4;
738 break;
739 /* dst = *(u32 *)(ul) (src + off) */
740 case BPF_LDX | BPF_MEM | BPF_W:
741 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
742 if (insn_is_zext(&insn[i + 1]))
743 addrs[++i] = ctx->idx * 4;
744 break;
745 /* dst = *(u64 *)(ul) (src + off) */
746 case BPF_LDX | BPF_MEM | BPF_DW:
747 PPC_BPF_LL(dst_reg, src_reg, off);
748 break;
749
750 /*
751 * Doubleword load
752 * 16 byte instruction that uses two 'struct bpf_insn'
753 */
754 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
755 imm64 = ((u64)(u32) insn[i].imm) |
756 (((u64)(u32) insn[i+1].imm) << 32);
757 /* Adjust for two bpf instructions */
758 addrs[++i] = ctx->idx * 4;
759 PPC_LI64(dst_reg, imm64);
760 break;
761
762 /*
763 * Return/Exit
764 */
765 case BPF_JMP | BPF_EXIT:
766 /*
767 * If this isn't the very last instruction, branch to
768 * the epilogue. If we _are_ the last instruction,
769 * we'll just fall through to the epilogue.
770 */
771 if (i != flen - 1)
772 PPC_JMP(exit_addr);
773 /* else fall through to the epilogue */
774 break;
775
776 /*
777 * Call kernel helper or bpf function
778 */
779 case BPF_JMP | BPF_CALL:
780 ctx->seen |= SEEN_FUNC;
781
782 ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass,
783 &func_addr, &func_addr_fixed);
784 if (ret < 0)
785 return ret;
786
787 if (func_addr_fixed)
788 bpf_jit_emit_func_call_hlp(image, ctx, func_addr);
789 else
790 bpf_jit_emit_func_call_rel(image, ctx, func_addr);
791 /* move return value from r3 to BPF_REG_0 */
792 EMIT(PPC_RAW_MR(b2p[BPF_REG_0], 3));
793 break;
794
795 /*
796 * Jumps and branches
797 */
798 case BPF_JMP | BPF_JA:
799 PPC_JMP(addrs[i + 1 + off]);
800 break;
801
802 case BPF_JMP | BPF_JGT | BPF_K:
803 case BPF_JMP | BPF_JGT | BPF_X:
804 case BPF_JMP | BPF_JSGT | BPF_K:
805 case BPF_JMP | BPF_JSGT | BPF_X:
806 case BPF_JMP32 | BPF_JGT | BPF_K:
807 case BPF_JMP32 | BPF_JGT | BPF_X:
808 case BPF_JMP32 | BPF_JSGT | BPF_K:
809 case BPF_JMP32 | BPF_JSGT | BPF_X:
810 true_cond = COND_GT;
811 goto cond_branch;
812 case BPF_JMP | BPF_JLT | BPF_K:
813 case BPF_JMP | BPF_JLT | BPF_X:
814 case BPF_JMP | BPF_JSLT | BPF_K:
815 case BPF_JMP | BPF_JSLT | BPF_X:
816 case BPF_JMP32 | BPF_JLT | BPF_K:
817 case BPF_JMP32 | BPF_JLT | BPF_X:
818 case BPF_JMP32 | BPF_JSLT | BPF_K:
819 case BPF_JMP32 | BPF_JSLT | BPF_X:
820 true_cond = COND_LT;
821 goto cond_branch;
822 case BPF_JMP | BPF_JGE | BPF_K:
823 case BPF_JMP | BPF_JGE | BPF_X:
824 case BPF_JMP | BPF_JSGE | BPF_K:
825 case BPF_JMP | BPF_JSGE | BPF_X:
826 case BPF_JMP32 | BPF_JGE | BPF_K:
827 case BPF_JMP32 | BPF_JGE | BPF_X:
828 case BPF_JMP32 | BPF_JSGE | BPF_K:
829 case BPF_JMP32 | BPF_JSGE | BPF_X:
830 true_cond = COND_GE;
831 goto cond_branch;
832 case BPF_JMP | BPF_JLE | BPF_K:
833 case BPF_JMP | BPF_JLE | BPF_X:
834 case BPF_JMP | BPF_JSLE | BPF_K:
835 case BPF_JMP | BPF_JSLE | BPF_X:
836 case BPF_JMP32 | BPF_JLE | BPF_K:
837 case BPF_JMP32 | BPF_JLE | BPF_X:
838 case BPF_JMP32 | BPF_JSLE | BPF_K:
839 case BPF_JMP32 | BPF_JSLE | BPF_X:
840 true_cond = COND_LE;
841 goto cond_branch;
842 case BPF_JMP | BPF_JEQ | BPF_K:
843 case BPF_JMP | BPF_JEQ | BPF_X:
844 case BPF_JMP32 | BPF_JEQ | BPF_K:
845 case BPF_JMP32 | BPF_JEQ | BPF_X:
846 true_cond = COND_EQ;
847 goto cond_branch;
848 case BPF_JMP | BPF_JNE | BPF_K:
849 case BPF_JMP | BPF_JNE | BPF_X:
850 case BPF_JMP32 | BPF_JNE | BPF_K:
851 case BPF_JMP32 | BPF_JNE | BPF_X:
852 true_cond = COND_NE;
853 goto cond_branch;
854 case BPF_JMP | BPF_JSET | BPF_K:
855 case BPF_JMP | BPF_JSET | BPF_X:
856 case BPF_JMP32 | BPF_JSET | BPF_K:
857 case BPF_JMP32 | BPF_JSET | BPF_X:
858 true_cond = COND_NE;
859 /* Fall through */
860
861 cond_branch:
862 switch (code) {
863 case BPF_JMP | BPF_JGT | BPF_X:
864 case BPF_JMP | BPF_JLT | BPF_X:
865 case BPF_JMP | BPF_JGE | BPF_X:
866 case BPF_JMP | BPF_JLE | BPF_X:
867 case BPF_JMP | BPF_JEQ | BPF_X:
868 case BPF_JMP | BPF_JNE | BPF_X:
869 case BPF_JMP32 | BPF_JGT | BPF_X:
870 case BPF_JMP32 | BPF_JLT | BPF_X:
871 case BPF_JMP32 | BPF_JGE | BPF_X:
872 case BPF_JMP32 | BPF_JLE | BPF_X:
873 case BPF_JMP32 | BPF_JEQ | BPF_X:
874 case BPF_JMP32 | BPF_JNE | BPF_X:
875 /* unsigned comparison */
876 if (BPF_CLASS(code) == BPF_JMP32)
877 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
878 else
879 EMIT(PPC_RAW_CMPLD(dst_reg, src_reg));
880 break;
881 case BPF_JMP | BPF_JSGT | BPF_X:
882 case BPF_JMP | BPF_JSLT | BPF_X:
883 case BPF_JMP | BPF_JSGE | BPF_X:
884 case BPF_JMP | BPF_JSLE | BPF_X:
885 case BPF_JMP32 | BPF_JSGT | BPF_X:
886 case BPF_JMP32 | BPF_JSLT | BPF_X:
887 case BPF_JMP32 | BPF_JSGE | BPF_X:
888 case BPF_JMP32 | BPF_JSLE | BPF_X:
889 /* signed comparison */
890 if (BPF_CLASS(code) == BPF_JMP32)
891 EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
892 else
893 EMIT(PPC_RAW_CMPD(dst_reg, src_reg));
894 break;
895 case BPF_JMP | BPF_JSET | BPF_X:
896 case BPF_JMP32 | BPF_JSET | BPF_X:
897 if (BPF_CLASS(code) == BPF_JMP) {
898 EMIT(PPC_RAW_AND_DOT(b2p[TMP_REG_1], dst_reg,
899 src_reg));
900 } else {
901 int tmp_reg = b2p[TMP_REG_1];
902
903 EMIT(PPC_RAW_AND(tmp_reg, dst_reg, src_reg));
904 EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg, 0, 0,
905 31));
906 }
907 break;
908 case BPF_JMP | BPF_JNE | BPF_K:
909 case BPF_JMP | BPF_JEQ | BPF_K:
910 case BPF_JMP | BPF_JGT | BPF_K:
911 case BPF_JMP | BPF_JLT | BPF_K:
912 case BPF_JMP | BPF_JGE | BPF_K:
913 case BPF_JMP | BPF_JLE | BPF_K:
914 case BPF_JMP32 | BPF_JNE | BPF_K:
915 case BPF_JMP32 | BPF_JEQ | BPF_K:
916 case BPF_JMP32 | BPF_JGT | BPF_K:
917 case BPF_JMP32 | BPF_JLT | BPF_K:
918 case BPF_JMP32 | BPF_JGE | BPF_K:
919 case BPF_JMP32 | BPF_JLE | BPF_K:
920 {
921 bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32;
922
923 /*
924 * Need sign-extended load, so only positive
925 * values can be used as imm in cmpldi
926 */
927 if (imm >= 0 && imm < 32768) {
928 if (is_jmp32)
929 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
930 else
931 EMIT(PPC_RAW_CMPLDI(dst_reg, imm));
932 } else {
933 /* sign-extending load */
934 PPC_LI32(b2p[TMP_REG_1], imm);
935 /* ... but unsigned comparison */
936 if (is_jmp32)
937 EMIT(PPC_RAW_CMPLW(dst_reg,
938 b2p[TMP_REG_1]));
939 else
940 EMIT(PPC_RAW_CMPLD(dst_reg,
941 b2p[TMP_REG_1]));
942 }
943 break;
944 }
945 case BPF_JMP | BPF_JSGT | BPF_K:
946 case BPF_JMP | BPF_JSLT | BPF_K:
947 case BPF_JMP | BPF_JSGE | BPF_K:
948 case BPF_JMP | BPF_JSLE | BPF_K:
949 case BPF_JMP32 | BPF_JSGT | BPF_K:
950 case BPF_JMP32 | BPF_JSLT | BPF_K:
951 case BPF_JMP32 | BPF_JSGE | BPF_K:
952 case BPF_JMP32 | BPF_JSLE | BPF_K:
953 {
954 bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32;
955
956 /*
957 * signed comparison, so any 16-bit value
958 * can be used in cmpdi
959 */
960 if (imm >= -32768 && imm < 32768) {
961 if (is_jmp32)
962 EMIT(PPC_RAW_CMPWI(dst_reg, imm));
963 else
964 EMIT(PPC_RAW_CMPDI(dst_reg, imm));
965 } else {
966 PPC_LI32(b2p[TMP_REG_1], imm);
967 if (is_jmp32)
968 EMIT(PPC_RAW_CMPW(dst_reg,
969 b2p[TMP_REG_1]));
970 else
971 EMIT(PPC_RAW_CMPD(dst_reg,
972 b2p[TMP_REG_1]));
973 }
974 break;
975 }
976 case BPF_JMP | BPF_JSET | BPF_K:
977 case BPF_JMP32 | BPF_JSET | BPF_K:
978 /* andi does not sign-extend the immediate */
979 if (imm >= 0 && imm < 32768)
980 /* PPC_ANDI is _only/always_ dot-form */
981 EMIT(PPC_RAW_ANDI(b2p[TMP_REG_1], dst_reg, imm));
982 else {
983 int tmp_reg = b2p[TMP_REG_1];
984
985 PPC_LI32(tmp_reg, imm);
986 if (BPF_CLASS(code) == BPF_JMP) {
987 EMIT(PPC_RAW_AND_DOT(tmp_reg, dst_reg,
988 tmp_reg));
989 } else {
990 EMIT(PPC_RAW_AND(tmp_reg, dst_reg,
991 tmp_reg));
992 EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg,
993 0, 0, 31));
994 }
995 }
996 break;
997 }
998 PPC_BCC(true_cond, addrs[i + 1 + off]);
999 break;
1000
1001 /*
1002 * Tail call
1003 */
1004 case BPF_JMP | BPF_TAIL_CALL:
1005 ctx->seen |= SEEN_TAILCALL;
1006 bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
1007 break;
1008
1009 default:
1010 /*
1011 * The filter contains something cruel & unusual.
1012 * We don't handle it, but also there shouldn't be
1013 * anything missing from our list.
1014 */
1015 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n",
1016 code, i);
1017 return -ENOTSUPP;
1018 }
1019 }
1020
1021 /* Set end-of-body-code address for exit. */
1022 addrs[i] = ctx->idx * 4;
1023
1024 return 0;
1025 }
1026
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
343
days inactive
343
days old
0 comments
1 participants
participants (1)
-
kernel test robot