diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index 163d38c0ba64..9c7ddb4d331f 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -514,3 +514,17 @@ Date: July 2021 Contact: "Daeho Jeong" Description: You can control for which gc mode the "gc_reclaimed_segments" node shows. Refer to the description of the modes in "gc_reclaimed_segments". + +What: /sys/fs/f2fs//hot_data_age_threshold +Date: November 2022 +Contact: "Ping Xiong" +Description: When DATA SEPARATION is on, it controls the age threshold to indicate + the data blocks as hot. By default it was initialized as 262144 blocks + (equals to 1GB). + +What: /sys/fs/f2fs//warm_data_age_threshold +Date: November 2022 +Contact: "Ping Xiong" +Description: When DATA SEPARATION is on, it controls the age threshold to indicate + the data blocks as warm. By default it was initialized as 2621440 blocks + (equals to 10GB). diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index c9fa1faf2aeb..c841b5208771 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3416,8 +3416,7 @@ difficult since unequal pointers can no longer be compared. However, if this command-line option is specified, then all normal pointers will have their true - value printed. Pointers printed via %pK may still be - hashed. This option should only be specified when + value printed. This option should only be specified when debugging the kernel. Please do not use on production kernels. diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml index 56c78317f9b7..57b9d2344b8a 100644 --- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml +++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml @@ -82,10 +82,9 @@ allOf: - mediatek,mt2701-smi-common then: properties: - clock: - items: - minItems: 3 - maxItems: 3 + clocks: + minItems: 3 + maxItems: 3 clock-names: items: - const: apb @@ -101,10 +100,9 @@ allOf: then: properties: - clock: - items: - minItems: 4 - maxItems: 4 + clocks: + minItems: 4 + maxItems: 4 clock-names: items: - const: apb @@ -114,10 +112,9 @@ allOf: else: # for gen2 HW that don't have gals properties: - clock: - items: - minItems: 2 - maxItems: 2 + clocks: + minItems: 2 + maxItems: 2 clock-names: items: - const: apb diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml index 06b623b34f48..2335a52f6f5a 100644 --- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml +++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml @@ -77,11 +77,11 @@ allOf: then: properties: - clock: - items: - minItems: 3 - maxItems: 3 + clocks: + minItems: 2 + maxItems: 3 clock-names: + minItems: 2 items: - const: apb - const: smi @@ -89,10 +89,9 @@ allOf: else: properties: - clock: - items: - minItems: 2 - maxItems: 2 + clocks: + minItems: 2 + maxItems: 2 clock-names: items: - const: apb @@ -106,7 +105,6 @@ allOf: - mediatek,mt2701-smi-larb - mediatek,mt2712-smi-larb - mediatek,mt6779-smi-larb - - mediatek,mt8167-smi-larb then: required: diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index b91e5a8444d5..f122a2b7c7d1 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -300,6 +300,10 @@ inlinecrypt When possible, encrypt/decrypt the contents of encrypted Documentation/block/inline-encryption.rst. atgc Enable age-threshold garbage collection, it provides high effectiveness and efficiency on background GC. +age_extent_cache Enable an age extent cache based on rb-tree. It records + data block update frequency of the extent per inode, in + order to provide better temperature hints for data block + allocation. ======================== ============================================================ Debugfs Entries diff --git a/Documentation/leds/well-known-leds.txt b/Documentation/leds/well-known-leds.txt new file mode 100644 index 000000000000..2160382c86be --- /dev/null +++ b/Documentation/leds/well-known-leds.txt @@ -0,0 +1,72 @@ +-*- org -*- + +It is somehow important to provide consistent interface to the +userland. LED devices have one problem there, and that is naming of +directories in /sys/class/leds. It would be nice if userland would +just know right "name" for given LED function, but situation got more +complex. + +Anyway, if backwards compatibility is not an issue, new code should +use one of the "good" names from this list, and you should extend the +list where applicable. + +Legacy names are listed, too; in case you are writing application that +wants to use particular feature, you should probe for good name, first, +but then try the legacy ones, too. + +Notice there's a list of functions in include/dt-bindings/leds/common.h . + +* Gamepads and joysticks + +Game controllers may feature LEDs to indicate a player number. This is commonly +used on game consoles in which multiple controllers can be connected to a system. +The "player LEDs" are then programmed with a pattern to indicate a particular +player. For example, a game controller with 4 LEDs, may be programmed with "x---" +to indicate player 1, "-x--" to indicate player 2 etcetera where "x" means on. +Input drivers can utilize the LED class to expose the individual player LEDs +of a game controller using the function "player". +Note: tracking and management of Player IDs is the responsibility of user space, +though drivers may pick a default value. + +Good: "input*:*:player-{1,2,3,4,5} + +* Keyboards + +Good: "input*:*:capslock" +Good: "input*:*:scrolllock" +Good: "input*:*:numlock" +Legacy: "shift-key-light" (Motorola Droid 4, capslock) + +Set of common keyboard LEDs, going back to PC AT or so. + +Legacy: "tpacpi::thinklight" (IBM/Lenovo Thinkpads) +Legacy: "lp5523:kb{1,2,3,4,5,6}" (Nokia N900) + +Frontlight/backlight of main keyboard. + +Legacy: "button-backlight" (Motorola Droid 4) + +Some phones have touch buttons below screen; it is different from main +keyboard. And this is their backlight. + +* Sound subsystem + +Good: "platform:*:mute" +Good: "platform:*:micmute" + +LEDs on notebook body, indicating that sound input / output is muted. + +* System notification + +Legacy: "status-led:{red,green,blue}" (Motorola Droid 4) +Legacy: "lp5523:{r,g,b}" (Nokia N900) + +Phones usually have multi-color status LED. + +* Power management + +Good: "platform:*:charging" (allwinner sun50i) + +* Screen + +Good: ":backlight" (Motorola Droid 4) diff --git a/Makefile b/Makefile index f564628b46fa..4a56f20505a3 100644 --- a/Makefile +++ b/Makefile @@ -796,6 +796,9 @@ stackp-flags-$(CONFIG_STACKPROTECTOR_STRONG) := -fstack-protector-strong KBUILD_CFLAGS += $(stackp-flags-y) +KBUILD_CFLAGS-$(CONFIG_WERROR) += -Werror +KBUILD_CFLAGS += $(KBUILD_CFLAGS-y) + ifdef CONFIG_CC_IS_CLANG KBUILD_CPPFLAGS += -Qunused-arguments KBUILD_CFLAGS += -Wno-format-invalid-specifier diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 6f4440e64311..9fb761ab3b74 100644 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -18,6 +18,7 @@ + @@ -254,6 +255,7 @@ + @@ -405,6 +407,8 @@ + + @@ -543,6 +547,7 @@ + @@ -814,6 +819,7 @@ + @@ -898,6 +904,7 @@ + @@ -953,6 +960,7 @@ + @@ -1126,6 +1134,7 @@ + @@ -1168,6 +1177,7 @@ + @@ -1708,6 +1718,7 @@ + @@ -2250,6 +2261,7 @@ + @@ -2488,6 +2500,8 @@ + + @@ -2551,6 +2565,10 @@ + + + + @@ -3367,6 +3385,7 @@ + @@ -3382,6 +3401,7 @@ + @@ -3654,6 +3674,7 @@ + @@ -3899,6 +3920,11 @@ + + + + + @@ -4131,9 +4157,25 @@ + + + + + + + + + + + + + + + + @@ -4261,6 +4303,7 @@ + @@ -4271,6 +4314,7 @@ + @@ -4334,14 +4378,17 @@ + + + @@ -4470,6 +4517,7 @@ + @@ -4495,6 +4543,7 @@ + @@ -4732,6 +4781,7 @@ + @@ -5906,6 +5956,7 @@ + @@ -6353,6 +6404,8 @@ + + @@ -6496,6 +6549,7 @@ + @@ -6786,6 +6840,7 @@ + @@ -7146,6 +7201,7 @@ + @@ -7270,6 +7326,9 @@ + + + @@ -10017,11 +10076,20 @@ + + + + + + + + + @@ -11778,6 +11846,14 @@ + + + + + + + + @@ -11964,6 +12040,17 @@ + + + + + + + + + + + @@ -13008,7 +13095,6 @@ - @@ -14349,18 +14435,18 @@ - + - + - + - + - + @@ -14683,12 +14769,12 @@ - + - + - + @@ -15123,6 +15209,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -15188,7 +15330,6 @@ - @@ -18431,10 +18572,7 @@ - - - - + @@ -19271,6 +19409,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -20098,9 +20262,9 @@ - + - + @@ -21293,6 +21457,20 @@ + + + + + + + + + + + + + + @@ -21832,6 +22010,14 @@ + + + + + + + + @@ -22110,11 +22296,6 @@ - - - - - @@ -22144,7 +22325,7 @@ - + @@ -22262,6 +22443,7 @@ + @@ -22534,222 +22716,222 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -23488,6 +23670,7 @@ + @@ -24086,17 +24269,6 @@ - - - - - - - - - - - @@ -25119,66 +25291,66 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -26151,7 +26323,7 @@ - + @@ -26457,12 +26629,12 @@ - + - + - + @@ -27475,6 +27647,7 @@ + @@ -27502,6 +27675,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -31717,6 +31910,7 @@ + @@ -33670,6 +33864,17 @@ + + + + + + + + + + + @@ -34282,6 +34487,9 @@ + + + @@ -34699,9 +34907,9 @@ - + - + @@ -38026,63 +38234,63 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -38882,6 +39090,14 @@ + + + + + + + + @@ -39917,6 +40133,17 @@ + + + + + + + + + + + @@ -40486,6 +40713,7 @@ + @@ -40784,6 +41012,14 @@ + + + + + + + + @@ -41374,6 +41610,7 @@ + @@ -41763,6 +42000,7 @@ + @@ -44349,81 +44587,81 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -44779,11 +45017,7 @@ - - - - - + @@ -46473,21 +46707,21 @@ - + - + - + - + - + - + @@ -46534,6 +46768,7 @@ + @@ -46646,6 +46881,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -48580,24 +48841,24 @@ - + - + - + - + - + - + - + @@ -48763,6 +49024,7 @@ + @@ -49023,6 +49285,7 @@ + @@ -49088,18 +49351,18 @@ - + - + - + - + - + @@ -50083,7 +50346,6 @@ - @@ -50669,6 +50931,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -51606,6 +51903,14 @@ + + + + + + + + @@ -52094,6 +52399,20 @@ + + + + + + + + + + + + + + @@ -53063,6 +53382,7 @@ + @@ -53503,13 +53823,13 @@ - + - + - + @@ -53654,11 +53974,6 @@ - - - - - @@ -54171,6 +54486,20 @@ + + + + + + + + + + + + + + @@ -54984,6 +55313,17 @@ + + + + + + + + + + + @@ -56908,114 +57248,114 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -58502,6 +58842,9 @@ + + + @@ -59151,6 +59494,7 @@ + @@ -60260,6 +60604,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -60775,24 +61157,24 @@ - + - + - + - + - + - + - + @@ -60813,6 +61195,14 @@ + + + + + + + + @@ -61037,96 +61427,96 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -62721,6 +63111,7 @@ + @@ -63898,6 +64289,7 @@ + @@ -64698,6 +65090,11 @@ + + + + + @@ -65649,14 +66046,6 @@ - - - - - - - - @@ -65673,6 +66062,7 @@ + @@ -66261,6 +66651,20 @@ + + + + + + + + + + + + + + @@ -66559,6 +66963,17 @@ + + + + + + + + + + + @@ -67204,6 +67619,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -68092,11 +68540,6 @@ - - - - - @@ -71976,6 +72419,14 @@ + + + + + + + + @@ -72197,7 +72648,6 @@ - @@ -72869,11 +73319,7 @@ - - - - - + @@ -75047,6 +75493,11 @@ + + + + + @@ -75686,6 +76137,7 @@ + @@ -75902,18 +76354,18 @@ - + - + - + - + - + @@ -77615,11 +78067,6 @@ - - - - - @@ -77752,6 +78199,14 @@ + + + + + + + + @@ -82126,7 +82581,6 @@ - @@ -83347,6 +83801,14 @@ + + + + + + + + @@ -84040,12 +84502,6 @@ - - - - - - @@ -85866,7 +86322,6 @@ - @@ -87307,15 +87762,15 @@ - + - + - + - + @@ -87755,36 +88210,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -88428,53 +88883,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -88780,6 +89188,15 @@ + + + + + + + + + @@ -89983,7 +90400,6 @@ - @@ -91223,7 +91639,7 @@ - + @@ -91277,6 +91693,7 @@ + @@ -92724,18 +93141,18 @@ - + - + - + - + - + @@ -93333,6 +93750,17 @@ + + + + + + + + + + + @@ -93807,6 +94235,7 @@ + @@ -94358,6 +94787,11 @@ + + + + + @@ -94517,6 +94951,14 @@ + + + + + + + + @@ -94989,6 +95431,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -95142,17 +95607,6 @@ - - - - - - - - - - - @@ -100050,6 +100504,7 @@ + @@ -101111,6 +101566,23 @@ + + + + + + + + + + + + + + + + + @@ -101223,21 +101695,21 @@ - + - + - + - + - + - + @@ -102493,6 +102965,14 @@ + + + + + + + + @@ -104372,21 +104852,21 @@ - + - + - + - + - + - + @@ -110684,6 +111164,14 @@ + + + + + + + + @@ -111364,6 +111852,14 @@ + + + + + + + + @@ -115393,36 +115889,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -115672,9 +116168,9 @@ - + - + @@ -115688,9 +116184,9 @@ - - - + + + @@ -115703,11 +116199,11 @@ - - - - - + + + + + @@ -115732,6 +116228,12 @@ + + + + + + @@ -116337,9 +116839,9 @@ - - - + + + @@ -116382,19 +116884,19 @@ - - - + + + - - - + + + - - - + + + @@ -116435,16 +116937,16 @@ - - - - + + + + - - - - + + + + @@ -116905,9 +117407,9 @@ - - - + + + @@ -117024,9 +117526,9 @@ - - - + + + @@ -117110,12 +117612,12 @@ - - - - - - + + + + + + @@ -117151,6 +117653,10 @@ + + + + @@ -117243,12 +117749,12 @@ - - + + - - + + @@ -117273,10 +117779,10 @@ - - - - + + + + @@ -118100,6 +118606,24 @@ + + + + + + + + + + + + + + + + + + @@ -118884,18 +119408,18 @@ - - - - - - + + + + + + - - - - + + + + @@ -118999,6 +119523,11 @@ + + + + + @@ -119232,11 +119761,11 @@ - - - - - + + + + + @@ -119463,9 +119992,9 @@ - - - + + + @@ -120191,6 +120720,8 @@ + + @@ -120315,8 +120846,8 @@ - - + + @@ -120334,6 +120865,7 @@ + @@ -120383,7 +120915,7 @@ - + @@ -120425,7 +120957,7 @@ - + @@ -120940,12 +121472,12 @@ - - - - - - + + + + + + @@ -121030,9 +121562,13 @@ - - - + + + + + + + @@ -121156,17 +121692,17 @@ - - - + + + - - - - - - + + + + + + @@ -121490,6 +122026,13 @@ + + + + + + + @@ -121710,8 +122253,8 @@ - - + + @@ -121741,8 +122284,8 @@ - - + + @@ -121758,15 +122301,15 @@ - - + + - - - - - + + + + + @@ -121779,6 +122322,10 @@ + + + + @@ -121788,8 +122335,8 @@ - - + + @@ -121797,9 +122344,9 @@ - - - + + + @@ -121826,13 +122373,13 @@ - - - + + + - - + + @@ -121862,9 +122409,9 @@ - - - + + + @@ -121877,10 +122424,10 @@ - - - - + + + + @@ -122743,6 +123290,10 @@ + + + + @@ -122923,9 +123474,9 @@ - - - + + + @@ -122956,14 +123507,19 @@ + + + + + - - + + @@ -122995,14 +123551,14 @@ - - - + + + - - - + + + @@ -123013,9 +123569,9 @@ - - - + + + @@ -123127,8 +123683,8 @@ - - + + @@ -123163,26 +123719,26 @@ - - - - - + + + + + - - - + + + - - + + - - - - + + + + @@ -124669,23 +125225,23 @@ - - - + + + - - + + - - - + + + - - - + + + @@ -125448,9 +126004,9 @@ - - - + + + @@ -125465,9 +126021,9 @@ - - - + + + @@ -125800,6 +126356,12 @@ + + + + + + @@ -125853,10 +126415,10 @@ - - - - + + + + @@ -126235,20 +126797,20 @@ - - + + - - + + - - + + - - + + @@ -126799,22 +127361,22 @@ - - + + - - - + + + - - + + - - - + + + @@ -128575,23 +129137,23 @@ - - + + - - - + + + - - - - + + + + @@ -128661,6 +129223,11 @@ + + + + + @@ -129669,10 +130236,10 @@ - - - - + + + + @@ -129814,13 +130381,13 @@ - - + + - - - + + + @@ -129913,6 +130480,16 @@ + + + + + + + + + + @@ -130102,15 +130679,15 @@ - + - - + + @@ -130127,10 +130704,10 @@ - - - - + + + + @@ -130156,9 +130733,9 @@ - - - + + + @@ -130247,7 +130824,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -130271,10 +130872,10 @@ - - - - + + + + @@ -130288,20 +130889,20 @@ - - + + - - + + - - + + - - + + @@ -130322,32 +130923,32 @@ - - - + + + - - - + + + - - - + + + - - - + + + @@ -130577,32 +131178,32 @@ - - - + + + - - - - + + + + - - - - + + + + - - - + + + - - - - + + + + @@ -130911,9 +131512,9 @@ - - - + + + @@ -130930,9 +131531,9 @@ - - - + + + @@ -130974,10 +131575,10 @@ - - - - + + + + @@ -130995,9 +131596,9 @@ - - - + + + @@ -131033,53 +131634,53 @@ - - + + - - + + - - + + - - - - + + + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - + + + + + @@ -131214,12 +131815,12 @@ - - + + - - + + @@ -131250,29 +131851,29 @@ - - - - - - + + + + + + - - - - - + + + + + - - - - - - - - + + + + + + + + @@ -131286,8 +131887,8 @@ - - + + @@ -131567,10 +132168,10 @@ - - - - + + + + @@ -131671,34 +132272,34 @@ - - - - - - + + + + + + - - - + + + - - - + + + - - + + @@ -131777,41 +132378,41 @@ - - + + - + - - + + - - + + - - + + - - - + + + - - + + - - - - - - + + + + + + @@ -131824,16 +132425,16 @@ - - - - - - + + + + + + - - + + @@ -133542,10 +134143,10 @@ - - - - + + + + @@ -133559,14 +134160,14 @@ - - - + + + - - - + + + @@ -133602,20 +134203,20 @@ - - - + + + - - - - + + + + - - - + + + @@ -133899,8 +134500,8 @@ - - + + @@ -133944,17 +134545,17 @@ - - - - - + + + + + - - - - + + + + @@ -133968,26 +134569,26 @@ - - - - - - + + + + + + - - - - - - - - + + + + + + + + - - + + @@ -134090,10 +134691,10 @@ - - - - + + + + @@ -134115,8 +134716,8 @@ - - + + @@ -134467,11 +135068,17 @@ + + + + + + @@ -134538,6 +135145,13 @@ + + + + + + + @@ -134914,8 +135528,8 @@ - - + + @@ -134946,7 +135560,7 @@ - + @@ -135321,24 +135935,24 @@ - - + + - - + + - - + + - - + + @@ -135377,26 +135991,26 @@ - - + + - - - + + + - - - - - + + + + + - - - - + + + + @@ -135453,8 +136067,8 @@ - - + + @@ -135585,9 +136199,9 @@ - - - + + + @@ -135595,8 +136209,8 @@ - - + + @@ -135865,6 +136479,11 @@ + + + + + @@ -136266,53 +136885,53 @@ - - - - + + + + - - - - + + + + - - + + - - - + + + - - - + + + - - + + - - + + - - - + + + - - - + + + - - - + + + @@ -136320,14 +136939,14 @@ - - - + + + - - - + + + @@ -136507,35 +137126,35 @@ - + - - - + + + - - - + + + - - - + + + - - - + + + - - + + @@ -137111,14 +137730,14 @@ - - - - + + + + - - + + @@ -137139,6 +137758,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -137171,24 +137815,24 @@ - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + @@ -138047,15 +138691,15 @@ - - - - + + + + - - - + + + @@ -138114,10 +138758,10 @@ - - - - + + + + @@ -138235,17 +138879,17 @@ - - - + + + - - + + @@ -138258,8 +138902,8 @@ - - + + @@ -138282,11 +138926,11 @@ - - - - - + + + + + @@ -138301,12 +138945,12 @@ - - + + - - + + @@ -138351,6 +138995,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -138361,10 +139068,23 @@ + + + + + + + + + + + + + @@ -138385,29 +139105,29 @@ - - - - - - + + + + + + - - - - - + + + + + - - - - - - - - + + + + + + + + @@ -138788,33 +139508,33 @@ - - - + + + - - - + + + - - - - + + + + - - + + - - + + - - - + + + @@ -139000,6 +139720,12 @@ + + + + + + @@ -139046,6 +139772,12 @@ + + + + + + @@ -139295,8 +140027,8 @@ - - + + @@ -139382,6 +140114,12 @@ + + + + + + @@ -139411,6 +140149,11 @@ + + + + + @@ -139425,6 +140168,12 @@ + + + + + + @@ -139568,33 +140317,33 @@ - - + + - - + + - - + + - - + + - - - - - + + + + + - - - - + + + + @@ -139719,8 +140468,8 @@ - - + + @@ -140020,6 +140769,11 @@ + + + + + @@ -140028,8 +140782,8 @@ - - + + @@ -140125,9 +140879,9 @@ - - - + + + @@ -140140,6 +140894,11 @@ + + + + + @@ -140198,8 +140957,8 @@ - - + + @@ -140220,12 +140979,12 @@ - - - - - - + + + + + + @@ -140266,12 +141025,12 @@ - - - - - - + + + + + + @@ -140304,13 +141063,13 @@ - - - - - - - + + + + + + + @@ -141088,10 +141847,10 @@ - - - - + + + + @@ -141336,6 +142095,10 @@ + + + + @@ -141715,45 +142478,45 @@ - - - - + + + + - - - - - + + + + + - - - + + + - - - - + + + + - - + + - - + + - - - + + + @@ -141780,9 +142543,9 @@ - - - + + + @@ -142099,11 +142862,11 @@ - + - - + + @@ -142169,16 +142932,16 @@ - - - - + + + + - - - - + + + + @@ -142234,18 +142997,18 @@ - - - - - + + + + + - - - - - + + + + + @@ -142257,8 +143020,8 @@ - - + + @@ -142268,9 +143031,9 @@ - - - + + + @@ -142278,11 +143041,11 @@ - - - - - + + + + + @@ -142299,11 +143062,11 @@ - - - - - + + + + + @@ -142311,12 +143074,12 @@ - - + + - - + + @@ -142325,9 +143088,9 @@ - - - + + + @@ -142341,49 +143104,49 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -142398,33 +143161,33 @@ - - - - - + + + + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + @@ -143022,16 +143785,16 @@ - - - + + + - - - - - + + + + + @@ -143336,9 +144099,9 @@ - - - + + + @@ -143365,9 +144128,9 @@ - - - + + + @@ -143375,9 +144138,9 @@ - - - + + + @@ -143395,9 +144158,9 @@ - - - + + + @@ -143405,9 +144168,9 @@ - - - + + + @@ -143438,10 +144201,10 @@ - - - - + + + + @@ -143539,9 +144302,9 @@ - - - + + + @@ -143549,9 +144312,9 @@ - - - + + + @@ -143968,9 +144731,9 @@ - - - + + + @@ -144015,9 +144778,9 @@ - - - + + + @@ -144341,15 +145104,15 @@ - - - - - + + + + + - - + + @@ -144634,29 +145397,29 @@ - - + + - - - + + + - - + + - - + + - - + + @@ -144669,36 +145432,36 @@ - - - + + + - - + + - - + + - - + + - - + + - - - - - + + + + + - - + + @@ -144926,10 +145689,10 @@ - - - - + + + + @@ -145063,16 +145826,16 @@ - - - - - + + + + + - - - + + + @@ -145721,21 +146484,21 @@ - - - + + + - - + + - - + + - - + + @@ -145745,13 +146508,13 @@ - - - + + + - - + + @@ -145760,28 +146523,28 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - + + @@ -145789,19 +146552,19 @@ - - - - - + + + + + - - - - - - + + + + + + @@ -145811,9 +146574,9 @@ - - - + + + @@ -145826,12 +146589,12 @@ - - + + - - + + @@ -145839,18 +146602,18 @@ - - + + - - + + - - - - + + + + @@ -145882,91 +146645,91 @@ - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - + + + + @@ -146043,12 +146806,12 @@ - - + + - - + + @@ -147346,11 +148109,11 @@ - - - - - + + + + + @@ -147420,6 +148183,14 @@ + + + + + + + + @@ -147630,11 +148401,11 @@ - - - - - + + + + + @@ -147651,11 +148422,11 @@ - - - - - + + + + + @@ -147735,9 +148506,9 @@ - - - + + + @@ -147842,9 +148613,9 @@ - - - + + + @@ -147855,14 +148626,14 @@ - - - + + + - - - + + + @@ -147896,12 +148667,12 @@ - - - - - - + + + + + + @@ -147909,8 +148680,8 @@ - - + + @@ -147932,8 +148703,8 @@ - - + + @@ -148349,22 +149120,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -148385,9 +149156,9 @@ - - - + + + @@ -148435,23 +149206,23 @@ - - - - + + + + - - - - + + + + - - - - - + + + + + @@ -148506,11 +149277,11 @@ - - - - - + + + + + @@ -148543,16 +149314,16 @@ - - - - + + + + - - - - + + + + diff --git a/android/abi_gki_aarch64_mtk b/android/abi_gki_aarch64_mtk index 1a3cc8ff7d30..8d2c95696079 100644 --- a/android/abi_gki_aarch64_mtk +++ b/android/abi_gki_aarch64_mtk @@ -412,6 +412,7 @@ devm_kasprintf devm_kfree devm_kmalloc + devm_led_classdev_flash_register_ext devm_led_classdev_register_ext devm_led_classdev_unregister devm_mbox_controller_register @@ -1149,7 +1150,9 @@ kvmalloc_node led_classdev_flash_register_ext led_classdev_flash_unregister + led_colors led_get_flash_fault + led_set_brightness led_set_brightness_sync led_set_flash_brightness led_set_flash_timeout @@ -1159,6 +1162,7 @@ led_update_brightness led_update_flash_brightness linear_range_get_max_value + linear_range_get_selector_high linear_range_get_value __list_add_valid __list_del_entry_valid @@ -1704,6 +1708,7 @@ regulator_enable regulator_enable_regmap regulator_get + regulator_get_bypass_regmap regulator_get_current_limit_regmap regulator_get_mode regulator_get_optional @@ -1721,6 +1726,7 @@ regulator_notifier_call_chain regulator_put regulator_set_active_discharge_regmap + regulator_set_bypass_regmap regulator_set_current_limit regulator_set_current_limit_regmap regulator_set_load @@ -2973,6 +2979,7 @@ platform_find_device_by_driver pm_wq power_supply_is_system_supplied + power_supply_register_no_ws power_supply_unreg_notifier prepare_to_wait printk_deferred diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index 4479f29e3b31..d1c65a2281cb 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -5,6 +5,7 @@ add_device_randomness add_memory add_memory_subsection + address_space_init_once add_swap_extent add_taint add_timer @@ -64,6 +65,8 @@ bio_add_pc_page bio_alloc_bioset bio_associate_blkg_from_css + __bio_crypt_clone + bio_crypt_set_ctx bio_endio bio_put bio_reset @@ -89,6 +92,7 @@ blk_execute_rq_nowait blk_get_request blk_mq_free_request + blk_mq_queue_inflight blk_mq_rq_cpu blk_mq_sched_mark_restart_hctx blk_mq_start_request @@ -2006,6 +2010,7 @@ preempt_schedule_notrace prepare_to_wait prepare_to_wait_event + prepare_to_wait_exclusive print_hex_dump printk printk_deferred @@ -2047,11 +2052,14 @@ qcom_smem_state_update_bits queue_delayed_work_on queue_work_on + radix_tree_delete_item radix_tree_gang_lookup radix_tree_insert radix_tree_iter_delete radix_tree_lookup + radix_tree_lookup_slot radix_tree_next_chunk + radix_tree_replace_slot radix_tree_preload ___ratelimit rational_best_approximation @@ -2289,6 +2297,7 @@ rtc_update_irq rtc_valid_tm rtnl_is_locked + __rtnl_link_register __rtnl_link_unregister rtnl_lock rtnl_unlock diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index dc16110241bc..f3e65159f870 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -129,6 +129,7 @@ cgroup_path_ns cgroup_taskset_first cgroup_taskset_next + check_move_unevictable_pages __check_object_size check_preempt_curr check_zeroed_user @@ -1659,6 +1660,7 @@ page_endio page_mapping __page_pinner_migration_failed + __pagevec_release panic panic_notifier_list panic_timeout @@ -2581,7 +2583,6 @@ __traceiter_android_vh_jiffies_update __traceiter_android_vh_logbuf __traceiter_android_vh_logbuf_pr_cont - __tracepoint_android_vh_madvise_cold_or_pageout __traceiter_android_vh_printk_hotplug __traceiter_android_vh_rproc_recovery __traceiter_android_vh_rproc_recovery_set @@ -2703,6 +2704,7 @@ __tracepoint_android_vh_jiffies_update __tracepoint_android_vh_logbuf __tracepoint_android_vh_logbuf_pr_cont + __tracepoint_android_vh_madvise_cold_or_pageout __tracepoint_android_vh_oom_check_panic __tracepoint_android_vh_printk_hotplug __tracepoint_android_vh_process_killed @@ -3026,6 +3028,7 @@ wait_for_completion_interruptible_timeout wait_for_completion_killable wait_for_completion_timeout + wait_on_page_bit __wait_rcu_gp wait_woken __wake_up diff --git a/android/abi_gki_aarch64_rockchip b/android/abi_gki_aarch64_rockchip index c1df5fc9548f..aabe57eaca5f 100644 --- a/android/abi_gki_aarch64_rockchip +++ b/android/abi_gki_aarch64_rockchip @@ -22,6 +22,8 @@ bdget_disk bdput _bin2bcd + __bitmap_and + __bitmap_andnot blk_cleanup_queue blk_execute_rq_nowait blk_mq_free_request @@ -80,10 +82,13 @@ __cfi_slowpath __check_object_size __class_create + class_create_file_ns class_destroy class_for_each_device __class_register + class_remove_file_ns class_unregister + __ClearPageMovable clk_bulk_disable clk_bulk_enable clk_bulk_prepare @@ -98,6 +103,7 @@ clk_hw_get_flags clk_hw_get_name clk_hw_get_parent + clk_hw_get_parent_by_index clk_hw_get_rate __clk_mux_determine_rate clk_notifier_register @@ -119,6 +125,7 @@ __const_udelay consume_skb cpu_bit_bitmap + cpufreq_cpu_get __cpufreq_driver_target cpufreq_generic_suspend cpufreq_register_governor @@ -167,18 +174,19 @@ crypto_unregister_shash crypto_unregister_template __crypto_xor + _ctype debugfs_attr_read debugfs_attr_write debugfs_create_dir debugfs_create_file debugfs_create_regset32 debugfs_remove + debugfs_rename default_llseek delayed_work_timer_fn del_gendisk del_timer del_timer_sync - desc_to_gpio destroy_workqueue dev_close dev_driver_string @@ -187,6 +195,7 @@ devfreq_add_governor devfreq_recommended_opp devfreq_register_opp_notifier + devfreq_remove_governor devfreq_resume_device devfreq_suspend_device devfreq_unregister_opp_notifier @@ -199,6 +208,7 @@ device_del device_destroy device_get_child_node_count + device_get_match_data device_get_named_child_node device_get_next_child_node device_initialize @@ -214,6 +224,7 @@ device_remove_file device_set_wakeup_capable device_set_wakeup_enable + device_unregister device_wakeup_enable _dev_info __dev_kfree_skb_any @@ -227,6 +238,7 @@ devm_devfreq_add_device devm_devfreq_event_add_edev devm_devfreq_register_opp_notifier + devm_device_add_group devm_extcon_dev_allocate devm_extcon_dev_register devm_free_irq @@ -284,6 +296,7 @@ devm_snd_soc_register_component devm_usb_get_phy _dev_notice + dev_open dev_pm_domain_detach dev_pm_opp_find_freq_ceil dev_pm_opp_find_freq_floor @@ -298,6 +311,7 @@ dev_pm_opp_register_set_opp_helper dev_pm_opp_set_rate dev_pm_opp_set_regulators + dev_pm_opp_set_supported_hw dev_pm_opp_unregister_set_opp_helper dev_printk devres_add @@ -360,8 +374,8 @@ driver_register driver_unregister drm_add_edid_modes + drm_add_modes_noedid drm_atomic_get_crtc_state - drm_atomic_get_new_bridge_state drm_atomic_get_new_connector_for_encoder drm_atomic_helper_bridge_destroy_state drm_atomic_helper_bridge_duplicate_state @@ -379,8 +393,12 @@ drm_compat_ioctl drm_connector_attach_encoder drm_connector_cleanup + drm_connector_has_possible_encoder drm_connector_init drm_connector_init_with_ddc + drm_connector_list_iter_begin + drm_connector_list_iter_end + drm_connector_list_iter_next drm_connector_unregister drm_connector_update_edid_property __drm_dbg @@ -398,7 +416,9 @@ drm_dp_aux_register drm_dp_aux_unregister drm_dp_bw_code_to_link_rate + drm_dp_channel_eq_ok drm_dp_dpcd_read + drm_dp_dpcd_read_link_status drm_dp_dpcd_write drm_dp_get_phy_test_pattern drm_dp_link_rate_to_bw_code @@ -468,6 +488,8 @@ enable_irq eth_mac_addr eth_platform_get_mac_address + ethtool_op_get_link + ethtool_op_get_ts_info eth_type_trans eth_validate_addr event_triggers_call @@ -479,6 +501,7 @@ extcon_set_state_sync extcon_unregister_notifier failure_tracking + fasync_helper fd_install find_next_bit find_next_zero_bit @@ -539,6 +562,7 @@ gpiod_get_value_cansleep gpiod_set_consumer_name gpiod_set_raw_value + gpiod_set_raw_value_cansleep gpiod_set_value gpiod_set_value_cansleep gpiod_to_irq @@ -568,6 +592,7 @@ hrtimer_start_range_ns i2c_adapter_type i2c_add_adapter + i2c_add_numbered_adapter i2c_del_adapter i2c_del_driver i2c_get_adapter @@ -583,6 +608,9 @@ i2c_smbus_xfer i2c_transfer i2c_transfer_buffer_flags + ida_alloc_range + ida_destroy + ida_free idr_alloc idr_destroy idr_find @@ -637,7 +665,6 @@ irq_find_mapping irq_get_irq_data irq_modify_status - irq_of_parse_and_map irq_set_affinity_hint irq_set_chained_handler_and_data irq_set_chip @@ -645,6 +672,7 @@ irq_set_chip_data irq_set_irq_type irq_set_irq_wake + irq_to_desc is_vmalloc_addr jiffies jiffies_to_msecs @@ -660,6 +688,7 @@ kfree_const kfree_sensitive kfree_skb + kill_fasync kimage_voffset __kmalloc kmalloc_caches @@ -681,6 +710,10 @@ kstrtouint_from_user kstrtoull kthread_create_on_node + kthread_create_worker + kthread_destroy_worker + kthread_flush_worker + kthread_queue_work kthread_should_stop kthread_stop ktime_get @@ -698,11 +731,15 @@ __list_add_valid __list_del_entry_valid __local_bh_enable_ip + __lock_page __log_post_read_mmio __log_read_mmio __log_write_mmio lzo1x_decompress_safe + mdiobus_alloc_size + mdiobus_free mdiobus_read + mdiobus_unregister mdiobus_write media_create_pad_link media_device_init @@ -717,7 +754,6 @@ media_pipeline_start media_pipeline_stop memcpy - __memcpy_fromio memdup_user memmove memset @@ -736,7 +772,9 @@ mipi_dsi_host_unregister misc_deregister misc_register + mmc_cqe_request_done mmc_of_parse + mmc_request_done __mmdrop mod_delayed_work_on mod_timer @@ -752,10 +790,14 @@ mutex_lock_interruptible mutex_trylock mutex_unlock + napi_gro_receive __netdev_alloc_skb netdev_err netdev_info + netdev_update_features netdev_warn + netif_carrier_off + netif_carrier_on netif_rx netif_rx_ni netif_tx_wake_queue @@ -788,6 +830,7 @@ of_device_is_available of_device_is_compatible of_drm_find_bridge + of_drm_find_panel of_find_compatible_node of_find_device_by_node of_find_i2c_device_by_node @@ -806,6 +849,7 @@ of_get_parent of_get_property of_get_regulator_init_data + of_graph_get_endpoint_by_regs of_graph_get_next_endpoint of_graph_get_remote_node of_graph_get_remote_port_parent @@ -849,6 +893,7 @@ perf_trace_buf_alloc perf_trace_run_bpf_submit pfn_valid + phy_attached_info phy_configure phy_drivers_register phy_drivers_unregister @@ -887,6 +932,7 @@ platform_driver_unregister platform_get_irq platform_get_irq_byname + platform_get_irq_optional platform_get_resource platform_get_resource_byname platform_irq_count @@ -913,6 +959,7 @@ power_supply_changed power_supply_class power_supply_get_battery_info + power_supply_get_by_name power_supply_get_by_phandle power_supply_get_drvdata power_supply_get_property @@ -936,6 +983,7 @@ put_disk __put_page __put_task_struct + put_unused_fd pwm_adjust_config pwm_apply_state queue_delayed_work_on @@ -946,6 +994,7 @@ _raw_spin_lock_bh _raw_spin_lock_irq _raw_spin_lock_irqsave + _raw_spin_trylock _raw_spin_unlock _raw_spin_unlock_bh _raw_spin_unlock_irq @@ -964,6 +1013,7 @@ __register_chrdev register_chrdev_region register_inetaddr_notifier + register_netdev register_netdevice register_netdevice_notifier register_pm_notifier @@ -1037,6 +1087,7 @@ scsi_ioctl_block_when_processing_errors sdev_prefix_printk sdhci_add_host + sdhci_execute_tuning sdhci_get_property sdhci_pltfm_clk_get_max_clock sdhci_pltfm_free @@ -1052,6 +1103,7 @@ seq_puts seq_read set_page_dirty_lock + __SetPageMovable sg_alloc_table sg_alloc_table_from_pages sg_free_table @@ -1068,6 +1120,7 @@ simple_strtoul single_open single_release + skb_add_rx_frag skb_clone skb_copy skb_copy_bits @@ -1084,6 +1137,7 @@ skcipher_walk_virt snd_pcm_format_width snd_soc_add_component_controls + snd_soc_add_dai_controls snd_soc_card_jack_new snd_soc_component_read snd_soc_component_set_jack @@ -1116,6 +1170,7 @@ snd_soc_pm_ops snd_soc_put_enum_double snd_soc_put_volsw + snd_soc_register_component snd_soc_unregister_component snprintf sort @@ -1125,6 +1180,7 @@ sscanf __stack_chk_fail __stack_chk_guard + strcasecmp strchr strcmp strcpy @@ -1146,6 +1202,7 @@ sync_file_create sync_file_get_fence synchronize_irq + synchronize_net synchronize_rcu syscon_node_to_regmap syscon_regmap_lookup_by_phandle @@ -1158,6 +1215,7 @@ sysfs_remove_link sysfs_streq system_freezable_wq + system_highpri_wq system_long_wq system_power_efficient_wq system_state @@ -1190,9 +1248,11 @@ typec_switch_register typec_switch_unregister __udelay + unlock_page __unregister_chrdev unregister_chrdev_region unregister_inetaddr_notifier + unregister_netdev unregister_netdevice_notifier unregister_netdevice_queue unregister_reboot_notifier @@ -1388,7 +1448,6 @@ # required by bcmdhd.ko alloc_etherdev_mqs complete_and_exit - dev_open down_interruptible down_timeout iwe_stream_add_event @@ -1399,9 +1458,6 @@ mmc_set_data_timeout mmc_sw_reset mmc_wait_for_req - netdev_update_features - netif_napi_add - __netif_napi_del netif_set_xps_queue __netlink_kernel_create netlink_kernel_release @@ -1410,7 +1466,6 @@ __nlmsg_put _raw_read_lock_bh _raw_read_unlock_bh - register_netdev sched_set_fifo_low sdio_claim_host sdio_disable_func @@ -1444,16 +1499,15 @@ strcat strspn sys_tz - unregister_netdev unregister_pm_notifier wireless_send_event # required by bifrost_kbase.ko __arch_clear_user - __bitmap_andnot __bitmap_equal __bitmap_or __bitmap_weight + __bitmap_xor cache_line_size clear_page complete_all @@ -1503,7 +1557,6 @@ simple_open strcspn system_freezing_cnt - system_highpri_wq _totalram_pages __traceiter_gpu_mem_total trace_output_call @@ -1515,9 +1568,6 @@ vmalloc_user vmf_insert_pfn_prot -# required by bq25700_charger.ko - power_supply_get_by_name - # required by cdc-wdm.ko cdc_parse_cdc_header @@ -1535,8 +1585,6 @@ # required by cfg80211.ko bpf_trace_run10 - _ctype - debugfs_rename dev_change_net_namespace __dev_get_by_index dev_get_by_index @@ -1569,7 +1617,6 @@ rfkill_blocked rfkill_pause_polling rfkill_resume_polling - skb_add_rx_frag __sock_create sock_release unregister_pernet_device @@ -1596,7 +1643,6 @@ # required by clk-rockchip-regmap.ko clk_hw_get_num_parents - clk_hw_get_parent_by_index divider_recalc_rate divider_round_rate_parent @@ -1608,6 +1654,7 @@ __clk_get_hw clk_hw_register_composite clk_hw_round_rate + clk_hw_set_parent clk_mux_ops clk_mux_ro_ops clk_register_divider_table @@ -1627,11 +1674,6 @@ # required by cm3218.ko i2c_smbus_write_word_data -# required by cma_heap.ko - cma_get_name - dma_heap_get_drvdata - dma_heap_put - # required by cpufreq-dt.ko cpufreq_enable_boost_support cpufreq_freq_attr_scaling_available_freqs @@ -1666,7 +1708,6 @@ # required by cqhci.ko devm_blk_ksm_init - mmc_cqe_request_done # required by cryptodev.ko crypto_ahash_final @@ -1677,7 +1718,11 @@ sg_last unregister_sysctl_table +# required by cw221x_battery.ko + power_supply_is_system_supplied + # required by display-connector.ko + drm_atomic_get_new_bridge_state drm_probe_ddc # required by dm9601.ko @@ -1695,7 +1740,6 @@ # required by dw-hdmi.ko drm_connector_attach_max_bpc_property drm_default_rgb_quant_range - of_graph_get_endpoint_by_regs # required by dw-mipi-dsi.ko drm_panel_bridge_add_typed @@ -1721,14 +1765,12 @@ mmc_regulator_set_ocr mmc_regulator_set_vqmmc mmc_remove_host - mmc_request_done sdio_signal_irq sg_miter_next sg_miter_start sg_miter_stop # required by dw_wdt.ko - platform_get_irq_optional watchdog_init_timeout watchdog_register_device watchdog_set_restart_priority @@ -1739,7 +1781,6 @@ bitmap_find_next_zero_area_off __bitmap_set phy_reset - _raw_spin_trylock usb_add_gadget_udc usb_del_gadget_udc usb_ep_set_maxpacket_limit @@ -1758,9 +1799,15 @@ usb_speed_string usb_wakeup_enabled_descendants +# required by dwmac-rockchip.ko + csum_tcpudp_nofold + ip_send_check + of_get_phy_mode + # required by fusb302.ko extcon_get_extcon_dev fwnode_create_software_node + sched_set_fifo tcpm_cc_change tcpm_pd_hard_reset tcpm_pd_receive @@ -1812,6 +1859,7 @@ i2c_verify_client # required by i2c-gpio.ko + desc_to_gpio i2c_bit_add_numbered_bus # required by i2c-hid.ko @@ -1822,7 +1870,6 @@ hid_parse_report # required by i2c-mux.ko - i2c_add_numbered_adapter __i2c_transfer rt_mutex_lock rt_mutex_trylock @@ -1880,7 +1927,6 @@ dev_fetch_sw_netstats dev_queue_xmit ether_setup - ethtool_op_get_link get_random_u32 __hw_addr_init __hw_addr_sync @@ -1889,10 +1935,7 @@ kernel_param_unlock kfree_skb_list ktime_get_seconds - napi_gro_receive netdev_set_default_ethtool_ops - netif_carrier_off - netif_carrier_on netif_receive_skb netif_receive_skb_list netif_tx_stop_all_queues @@ -1918,7 +1961,6 @@ skb_queue_head skb_queue_purge skb_queue_tail - synchronize_net unregister_inet6addr_notifier unregister_netdevice_many @@ -1955,9 +1997,6 @@ dev_pm_qos_expose_latency_tolerance dev_pm_qos_hide_latency_tolerance dev_pm_qos_update_user_latency_tolerance - ida_alloc_range - ida_destroy - ida_free init_srcu_struct memchr_inv param_ops_ulong @@ -2049,7 +2088,6 @@ __arm_smccc_hvc bus_for_each_dev device_register - device_unregister free_pages_exact memremap memunmap @@ -2076,6 +2114,7 @@ # required by pcie-dw-rockchip.ko cpumask_next_and + debugfs_create_devm_seqfile dw_pcie_find_ext_capability dw_pcie_host_init dw_pcie_link_up @@ -2106,7 +2145,6 @@ extcon_sync # required by phy-rockchip-inno-usb3.ko - strcasecmp usb_add_phy # required by phy-rockchip-samsung-hdptx-hdmi.ko @@ -2153,10 +2191,25 @@ clk_bulk_put of_genpd_add_provider_onecell panic + param_get_bool + param_set_bool pm_clk_add_clk pm_genpd_add_subdomain pm_genpd_init pm_genpd_remove + pm_wq + +# required by pps_core.ko + kobject_get + +# required by ptp.ko + kthread_cancel_delayed_work_sync + kthread_delayed_work_timer_fn + kthread_mod_delayed_work + kthread_queue_delayed_work + ktime_get_snapshot + posix_clock_register + posix_clock_unregister # required by pwm-regulator.ko regulator_map_voltage_iterate @@ -2173,6 +2226,13 @@ pwm_free pwm_request +# required by pwrseq_simple.ko + bitmap_alloc + devm_gpiod_get_array + gpiod_set_array_value_cansleep + mmc_pwrseq_register + mmc_pwrseq_unregister + # required by reboot-mode.ko devres_release kernel_kobj @@ -2186,6 +2246,7 @@ alloc_iova_fast dma_fence_wait_timeout free_iova_fast + idr_alloc_cyclic kstrdup_quotable_cmdline mmput @@ -2194,9 +2255,6 @@ irq_domain_xlate_onetwocell irq_set_parent -# required by rk628_dsi.ko - of_drm_find_panel - # required by rk805-pwrkey.ko devm_request_any_context_irq @@ -2221,6 +2279,10 @@ # required by rk860x-regulator.ko regulator_suspend_enable +# required by rk_cma_heap.ko + dma_heap_get_drvdata + dma_heap_put + # required by rk_crypto.ko crypto_ahash_digest crypto_dequeue_request @@ -2249,8 +2311,15 @@ # required by rk_ircut.ko drain_workqueue +# required by rk_system_heap.ko + deferred_free + dmabuf_page_pool_alloc + dmabuf_page_pool_create + dmabuf_page_pool_destroy + dmabuf_page_pool_free + swiotlb_max_segment + # required by rk_vcodec.ko - devfreq_remove_governor devm_iounmap dev_pm_domain_attach dev_pm_opp_get_freq @@ -2260,9 +2329,7 @@ __fdget iommu_device_unregister iommu_dma_reserve_iova - kthread_flush_worker __kthread_init_worker - kthread_queue_work kthread_worker_fn of_device_alloc of_dma_configure_id @@ -2287,7 +2354,18 @@ # required by rockchip-cpufreq.ko cpufreq_unregister_notifier dev_pm_opp_put_prop_name - dev_pm_opp_set_supported_hw + +# required by rockchip-hdmirx.ko + cec_s_phys_addr_from_edid + cpu_latency_qos_remove_request + device_create_with_groups + of_reserved_mem_device_release + v4l2_ctrl_log_status + v4l2_ctrl_subscribe_event + v4l2_find_dv_timings_cap + v4l2_src_change_event_subscribe + vb2_dma_contig_memops + vb2_fop_read # required by rockchip-rng.ko devm_hwrng_register @@ -2296,8 +2374,10 @@ # required by rockchip_bus.ko cpu_topology +# required by rockchip_debug.ko + nr_irqs + # required by rockchip_dmc.ko - cpufreq_cpu_get cpufreq_cpu_put cpufreq_quick_get devfreq_event_disable_edev @@ -2330,7 +2410,6 @@ regulator_get_linear_step # required by rockchip_pwm_remotectl.ko - irq_to_desc __tasklet_hi_schedule # required by rockchip_saradc.ko @@ -2368,6 +2447,7 @@ drm_atomic_commit drm_atomic_get_connector_state drm_atomic_get_plane_state + drm_atomic_helper_bridge_propagate_bus_fmt drm_atomic_helper_check drm_atomic_helper_check_plane_state drm_atomic_helper_cleanup_planes @@ -2398,8 +2478,9 @@ drm_atomic_set_mode_for_crtc drm_atomic_state_alloc __drm_atomic_state_free + drm_bridge_chain_mode_set drm_bridge_get_edid - drm_connector_has_possible_encoder + drm_connector_attach_content_protection_property drm_connector_list_iter_begin drm_connector_list_iter_end drm_connector_list_iter_next @@ -2416,9 +2497,7 @@ drm_crtc_vblank_put drm_debugfs_create_files drm_do_get_edid - drm_dp_channel_eq_ok drm_dp_clock_recovery_ok - drm_dp_dpcd_read_link_status drm_dp_get_adjust_request_pre_emphasis drm_dp_get_adjust_request_voltage drm_dp_read_desc @@ -2450,6 +2529,7 @@ drm_gem_unmap_dma_buf drm_get_format_info drm_get_format_name + drm_hdcp_update_content_protection drm_helper_mode_fill_fb_struct drm_kms_helper_poll_enable drm_kms_helper_poll_fini @@ -2475,6 +2555,7 @@ drm_mode_prune_invalid drm_mode_set_crtcinfo drm_modeset_lock_all + drm_modeset_unlock drm_modeset_unlock_all drm_mode_sort drm_mode_validate_size @@ -2488,6 +2569,7 @@ drm_plane_create_zpos_property drm_prime_get_contiguous_size __drm_printfn_seq_file + drm_property_blob_put drm_property_create drm_property_create_bitmask drm_property_create_bool @@ -2495,6 +2577,8 @@ drm_property_create_object drm_property_create_range drm_property_destroy + drm_property_lookup_blob + drm_property_replace_blob __drm_puts_seq_file drm_rect_calc_hscale drm_self_refresh_helper_cleanup @@ -2536,23 +2620,18 @@ sdhci_cqe_irq sdhci_dumpregs sdhci_enable_clk - sdhci_execute_tuning sdhci_pltfm_unregister sdhci_set_power_and_bus_voltage sdhci_set_uhs_signaling sdhci_setup_host # required by sdhci-of-dwcmshc.ko - device_get_match_data devm_clk_bulk_get_optional dma_get_required_mask sdhci_adma_write_desc sdhci_remove_host sdhci_request - -# required by sensor_dev.ko - class_create_file_ns - class_remove_file_ns + sdhci_reset_tuning # required by sg.ko blk_get_request @@ -2561,10 +2640,8 @@ blk_verify_command cdev_alloc class_interface_unregister - fasync_helper get_sg_io_hdr import_iovec - kill_fasync put_sg_io_hdr _raw_read_lock_irqsave _raw_read_unlock_irqrestore @@ -2592,12 +2669,7 @@ # required by smsc95xx.ko csum_partial - ethtool_op_get_ts_info - mdiobus_alloc_size - mdiobus_free __mdiobus_register - mdiobus_unregister - phy_attached_info phy_connect_direct phy_disconnect phy_ethtool_get_link_ksettings @@ -2615,9 +2687,6 @@ # required by snd-soc-es8316.ko snd_pcm_hw_constraint_list -# required by snd-soc-es8326.ko - snd_soc_register_component - # required by snd-soc-hdmi-codec.ko snd_ctl_add snd_ctl_new1 @@ -2626,7 +2695,7 @@ snd_pcm_fill_iec958_consumer snd_pcm_fill_iec958_consumer_hw_params snd_pcm_hw_constraint_eld - snd_pcm_stop_xrun + snd_pcm_stop # required by snd-soc-rk817.ko snd_soc_component_exit_regmap @@ -2637,7 +2706,8 @@ # required by snd-soc-rockchip-i2s-tdm.ko clk_is_match - snd_soc_add_dai_controls + pm_runtime_forbid + snd_pcm_stop_xrun # required by snd-soc-rockchip-i2s.ko of_prop_next_string @@ -2647,8 +2717,10 @@ snd_soc_jack_add_zones snd_soc_jack_get_type +# required by snd-soc-rockchip-spdif.ko + snd_pcm_create_iec958_consumer_hw_params + # required by snd-soc-rt5640.ko - gpiod_set_raw_value_cansleep regmap_register_patch snd_soc_dapm_force_bias_level @@ -2676,19 +2748,75 @@ spi_setup stream_open +# required by stmmac-platform.ko + device_get_phy_mode + of_get_mac_address + of_phy_is_fixed_link + platform_get_irq_byname_optional + +# required by stmmac.ko + devm_alloc_etherdev_mqs + dql_completed + dql_reset + ethtool_convert_legacy_u32_to_link_mode + ethtool_convert_link_mode_to_legacy_u32 + flow_block_cb_setup_simple + flow_rule_match_basic + flow_rule_match_ipv4_addrs + flow_rule_match_ports + mdiobus_get_phy + __napi_alloc_skb + napi_complete_done + napi_disable + __napi_schedule + napi_schedule_prep + netdev_alert + netdev_pick_tx + netdev_rss_key_fill + netif_device_attach + netif_device_detach + netif_napi_add + __netif_napi_del + netif_schedule_queue + netif_set_real_num_rx_queues + netif_set_real_num_tx_queues + of_mdiobus_register + page_pool_alloc_pages + page_pool_create + page_pool_destroy + page_pool_put_page + page_pool_release_page + phy_init_eee + phylink_connect_phy + phylink_create + phylink_destroy + phylink_disconnect_phy + phylink_ethtool_get_eee + phylink_ethtool_get_pauseparam + phylink_ethtool_get_wol + phylink_ethtool_ksettings_get + phylink_ethtool_ksettings_set + phylink_ethtool_nway_reset + phylink_ethtool_set_eee + phylink_ethtool_set_pauseparam + phylink_ethtool_set_wol + phylink_get_eee_err + phylink_mac_change + phylink_mii_ioctl + phylink_of_phy_connect + phylink_set_port_modes + phylink_speed_down + phylink_speed_up + phylink_start + phylink_stop + pm_wakeup_dev_event + reset_control_reset + skb_tstamp_tx + # required by sw_sync.ko dma_fence_free dma_fence_signal_locked __get_task_comm - put_unused_fd - -# required by system_heap.ko - deferred_free - dmabuf_page_pool_alloc - dmabuf_page_pool_create - dmabuf_page_pool_destroy - dmabuf_page_pool_free - swiotlb_max_segment # required by tcpci_husb311.ko tcpci_get_tcpm_port @@ -2712,6 +2840,7 @@ # required by timer-rockchip.ko clockevents_config_and_register + irq_of_parse_and_map # required by tps65132-regulator.ko regulator_set_active_discharge_regmap @@ -2792,6 +2921,7 @@ # required by video_rkisp.ko media_device_cleanup + __memcpy_fromio __memcpy_toio param_ops_ullong v4l2_pipeline_link_notify @@ -2831,7 +2961,6 @@ # required by zsmalloc.ko alloc_anon_inode - __ClearPageMovable contig_page_data dec_zone_page_state inc_zone_page_state @@ -2840,11 +2969,8 @@ kern_mount kern_unmount kill_anon_super - __lock_page page_mapping _raw_read_lock _raw_read_unlock _raw_write_lock _raw_write_unlock - __SetPageMovable - unlock_page diff --git a/android/abi_gki_aarch64_vivo b/android/abi_gki_aarch64_vivo index c3bc87e17c9f..28c4b05aa43f 100644 --- a/android/abi_gki_aarch64_vivo +++ b/android/abi_gki_aarch64_vivo @@ -277,6 +277,7 @@ del_gendisk del_timer del_timer_sync + dentry_path_raw desc_to_gpio destroy_workqueue dev_coredumpv diff --git a/android/abi_gki_aarch64_xiaomi b/android/abi_gki_aarch64_xiaomi index a162683958cd..ebbbb0e81420 100644 --- a/android/abi_gki_aarch64_xiaomi +++ b/android/abi_gki_aarch64_xiaomi @@ -200,3 +200,11 @@ wakeup_sources_read_unlock wakeup_sources_walk_start wakeup_sources_walk_next + +#required by mi_mempool.ko module + __traceiter_android_vh_mmput + __tracepoint_android_vh_mmput + __traceiter_android_vh_alloc_pages_reclaim_bypass + __tracepoint_android_vh_alloc_pages_reclaim_bypass + __traceiter_android_vh_alloc_pages_failure_bypass + __tracepoint_android_vh_alloc_pages_failure_bypass diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 3fc06f9e9bf0..4b8b2e0ecfab 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -687,6 +687,7 @@ config ARM64_ERRATUM_1508412 config ARM64_ERRATUM_2051678 bool "Cortex-A510: 2051678: disable Hardware Update of the page table's dirty bit" + default y help This options adds the workaround for ARM Cortex-A510 erratum ARM64_ERRATUM_2051678. Affected Coretex-A510 might not respect the ordering rules for diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms index e6a2f21ba3f8..6e6198fbf17b 100644 --- a/arch/arm64/Kconfig.platforms +++ b/arch/arm64/Kconfig.platforms @@ -152,6 +152,7 @@ config ARCH_MEDIATEK config ARCH_MESON bool "Amlogic Platforms" + select COMMON_CLK help This enables support for the arm64 based Amlogic SoCs such as the s905, S905X/D, S912, A113X/D or S905X/D2 diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 73696b1d7160..2ff6b13c9fe6 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -513,6 +513,7 @@ CONFIG_MMC_CRYPTO=y CONFIG_MMC_SDHCI=y CONFIG_MMC_SDHCI_PLTFM=y CONFIG_LEDS_CLASS_FLASH=y +CONFIG_LEDS_CLASS_MULTICOLOR=y CONFIG_LEDS_TRIGGER_TIMER=y CONFIG_LEDS_TRIGGER_TRANSIENT=y CONFIG_EDAC=y diff --git a/arch/arm64/configs/rockchip_gki.fragment b/arch/arm64/configs/rockchip_gki.fragment index 6253108101a8..1eb35920205b 100644 --- a/arch/arm64/configs/rockchip_gki.fragment +++ b/arch/arm64/configs/rockchip_gki.fragment @@ -50,10 +50,16 @@ CONFIG_DRM_PANEL_SIMPLE=m CONFIG_DRM_RK1000_TVE=m CONFIG_DRM_RK630_TVE=m CONFIG_DRM_ROCKCHIP=m +CONFIG_DRM_ROCKCHIP_RK618=m CONFIG_DRM_ROCKCHIP_RK628=m CONFIG_DRM_ROHM_BU18XL82=m CONFIG_DRM_SII902X=m CONFIG_DTC_SYMBOLS=y +# CONFIG_DWMAC_GENERIC is not set +# CONFIG_DWMAC_IPQ806X is not set +# CONFIG_DWMAC_QCOM_ETHQOS is not set +# CONFIG_DWMAC_SUN8I is not set +# CONFIG_DWMAC_SUNXI is not set CONFIG_DW_WATCHDOG=m CONFIG_GPIO_ROCKCHIP=m CONFIG_GREENASIA_FF=y @@ -146,6 +152,7 @@ CONFIG_MALI_BIFROST_EXPERT=y CONFIG_MALI_CSF_SUPPORT=y CONFIG_MALI_PLATFORM_NAME="rk" CONFIG_MALI_PWRSOFT_765=y +CONFIG_MFD_RK618=m CONFIG_MFD_RK628=m CONFIG_MFD_RK630_I2C=m CONFIG_MFD_RK806_SPI=m @@ -186,6 +193,7 @@ CONFIG_PROXIMITY_DEVICE=m CONFIG_PS_STK3410=m CONFIG_PS_UCS14620=m CONFIG_PWM_ROCKCHIP=m +CONFIG_PWRSEQ_SIMPLE=m CONFIG_REGULATOR_ACT8865=m CONFIG_REGULATOR_FAN53555=m CONFIG_REGULATOR_GPIO=m @@ -236,6 +244,7 @@ CONFIG_ROCKCHIP_PM_DOMAINS=m CONFIG_ROCKCHIP_PVTM=m CONFIG_ROCKCHIP_REMOTECTL=m CONFIG_ROCKCHIP_REMOTECTL_PWM=m +CONFIG_ROCKCHIP_MULTI_RGA=m CONFIG_ROCKCHIP_RGB=y CONFIG_ROCKCHIP_RKNPU=m CONFIG_ROCKCHIP_SARADC=m @@ -278,6 +287,7 @@ CONFIG_SND_SOC_RT5640=m CONFIG_SND_SOC_SPDIF=m CONFIG_SPI_ROCKCHIP=m CONFIG_SPI_SPIDEV=m +CONFIG_STMMAC_ETH=m CONFIG_SW_SYNC=m CONFIG_SYSCON_REBOOT_MODE=m CONFIG_TEE=m @@ -328,8 +338,8 @@ CONFIG_VIDEO_RK628_BT1120=m CONFIG_VIDEO_RK628_CSI=m CONFIG_VIDEO_RK_IRCUT=m CONFIG_VIDEO_ROCKCHIP_CIF=m +CONFIG_VIDEO_ROCKCHIP_HDMIRX=m CONFIG_VIDEO_ROCKCHIP_ISP=m -CONFIG_VIDEO_ROCKCHIP_ISPP=m CONFIG_VIDEO_S5K3L6XX=m CONFIG_VIDEO_S5KJN1=m CONFIG_VIDEO_SGM3784=m diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h index a074459f8f2f..7c2181c72116 100644 --- a/arch/arm64/include/asm/cache.h +++ b/arch/arm64/include/asm/cache.h @@ -6,6 +6,7 @@ #define __ASM_CACHE_H #include +#include #define CTR_L1IP_SHIFT 14 #define CTR_L1IP_MASK 3 @@ -49,15 +50,21 @@ */ #define ARCH_DMA_MINALIGN (128) -#ifdef CONFIG_KASAN_SW_TAGS -#define ARCH_SLAB_MINALIGN (1ULL << KASAN_SHADOW_SCALE_SHIFT) -#elif defined(CONFIG_KASAN_HW_TAGS) -#define ARCH_SLAB_MINALIGN MTE_GRANULE_SIZE -#endif - #ifndef __ASSEMBLY__ #include +#include + +#ifdef CONFIG_KASAN_SW_TAGS +#define ARCH_SLAB_MINALIGN (1ULL << KASAN_SHADOW_SCALE_SHIFT) +#elif defined(CONFIG_KASAN_HW_TAGS) +static inline unsigned int arch_slab_minalign(void) +{ + return kasan_hw_tags_enabled() ? MTE_GRANULE_SIZE : + __alignof__(unsigned long long); +} +#define arch_slab_minalign() arch_slab_minalign() +#endif #define ICACHEF_ALIASING 0 #define ICACHEF_VPIPT 1 diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index bda918948471..692b234255e5 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -107,7 +107,7 @@ msr_s SYS_ICC_SRE_EL2, x0 isb // Make sure SRE is now set mrs_s x0, SYS_ICC_SRE_EL2 // Read SRE back, - tbz x0, #0, 1f // and check that it sticks + tbz x0, #0, .Lskip_gicv3_\@ // and check that it sticks msr_s SYS_ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults .Lskip_gicv3_\@: .endm diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 4e9ee0676b53..b7fa62950b78 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -780,6 +780,10 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); #define kvm_vcpu_has_pmu(vcpu) \ (test_bit(KVM_ARM_VCPU_PMU_V3, (vcpu)->arch.features)) +#define kvm_supports_32bit_el0() \ + (system_supports_32bit_el0() && \ + !static_branch_unlikely(&arm64_mismatched_32bit_el0)) + int kvm_trng_call(struct kvm_vcpu *vcpu); #ifdef CONFIG_KVM extern phys_addr_t hyp_mem_base; diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h index 41c48c1ba610..6537d2deb728 100644 --- a/arch/arm64/include/asm/mte-kasan.h +++ b/arch/arm64/include/asm/mte-kasan.h @@ -5,6 +5,8 @@ #ifndef __ASM_MTE_KASAN_H #define __ASM_MTE_KASAN_H +#include +#include #include #ifndef __ASSEMBLY__ diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h index 8f1661603b78..b9ba19dbdb69 100644 --- a/arch/arm64/include/asm/percpu.h +++ b/arch/arm64/include/asm/percpu.h @@ -10,6 +10,7 @@ #include #include #include +#include static inline void set_my_cpu_offset(unsigned long off) { diff --git a/arch/arm64/include/asm/spectre.h b/arch/arm64/include/asm/spectre.h index c04d01dd4457..71060c1c23fe 100644 --- a/arch/arm64/include/asm/spectre.h +++ b/arch/arm64/include/asm/spectre.h @@ -67,7 +67,8 @@ struct bp_hardening_data { DECLARE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data); -static inline void arm64_apply_bp_hardening(void) +/* Called during entry so must be __always_inline */ +static __always_inline void arm64_apply_bp_hardening(void) { struct bp_hardening_data *d; diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index a40d98d5d156..428cfabd11c4 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -233,17 +233,20 @@ static void install_bp_hardening_cb(bp_hardening_cb_t fn) __this_cpu_write(bp_hardening_data.slot, HYP_VECTOR_SPECTRE_DIRECT); } -static void call_smc_arch_workaround_1(void) +/* Called during entry so must be noinstr */ +static noinstr void call_smc_arch_workaround_1(void) { arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); } -static void call_hvc_arch_workaround_1(void) +/* Called during entry so must be noinstr */ +static noinstr void call_hvc_arch_workaround_1(void) { arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); } -static void qcom_link_stack_sanitisation(void) +/* Called during entry so must be noinstr */ +static noinstr void qcom_link_stack_sanitisation(void) { u64 tmp; diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 67749fd3604d..78550c856c24 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -695,8 +695,7 @@ static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu) if (likely(!vcpu_mode_is_32bit(vcpu))) return false; - return !system_supports_32bit_el0() || - static_branch_unlikely(&arm64_mismatched_32bit_el0); + return !kvm_supports_32bit_el0(); } /** diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index c763808cacdf..27b783a711bb 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -225,7 +225,7 @@ static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) u64 mode = (*(u64 *)valp) & PSR_AA32_MODE_MASK; switch (mode) { case PSR_AA32_MODE_USR: - if (!system_supports_32bit_el0()) + if (!kvm_supports_32bit_el0()) return -EINVAL; break; case PSR_AA32_MODE_FIQ: diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index cebe39f3b1b6..cb0054ea7332 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -240,6 +240,14 @@ int handle_exit(struct kvm_vcpu *vcpu, int exception_index) { struct kvm_run *run = vcpu->run; + if (ARM_SERROR_PENDING(exception_index)) { + /* + * The SError is handled by handle_exit_early(). If the guest + * survives it will re-execute the original instruction. + */ + return 1; + } + exception_index = ARM_EXCEPTION_CODE(exception_index); switch (exception_index) { diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c index 73629094f903..0a99cb4e0c38 100644 --- a/arch/arm64/kvm/hyp/exception.c +++ b/arch/arm64/kvm/hyp/exception.c @@ -38,7 +38,10 @@ static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg) static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, u64 val) { - write_sysreg_el1(val, SYS_SPSR); + if (has_vhe()) + write_sysreg_el1(val, SYS_SPSR); + else + __vcpu_sys_reg(vcpu, SPSR_EL1) = val; } static void __vcpu_write_spsr_abt(struct kvm_vcpu *vcpu, u64 val) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 16807a46ab9f..a22555212681 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -423,7 +423,8 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR); - if (ARM_SERROR_PENDING(*exit_code)) { + if (ARM_SERROR_PENDING(*exit_code) && + ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) { u8 esr_ec = kvm_vcpu_trap_get_class(vcpu); /* diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 187576986aef..b9bb810991c5 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -618,7 +618,7 @@ static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) */ val = ((pmcr & ~ARMV8_PMU_PMCR_MASK) | (ARMV8_PMU_PMCR_MASK & 0xdecafbad)) & (~ARMV8_PMU_PMCR_E); - if (!system_supports_32bit_el0()) + if (!kvm_supports_32bit_el0()) val |= ARMV8_PMU_PMCR_LC; __vcpu_sys_reg(vcpu, r->reg) = val; } @@ -667,7 +667,7 @@ static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, val = __vcpu_sys_reg(vcpu, PMCR_EL0); val &= ~ARMV8_PMU_PMCR_MASK; val |= p->regval & ARMV8_PMU_PMCR_MASK; - if (!system_supports_32bit_el0()) + if (!kvm_supports_32bit_el0()) val |= ARMV8_PMU_PMCR_LC; __vcpu_sys_reg(vcpu, PMCR_EL0) = val; kvm_pmu_handle_pmcr(vcpu, val); diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index 7b8158ae36ec..83a0b6f9960b 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -228,6 +228,8 @@ SYM_FUNC_END_PI(__dma_flush_area) * - dir - DMA direction */ SYM_FUNC_START_PI(__dma_map_area) + cmp w2, #DMA_FROM_DEVICE + b.eq __dma_flush_area b __dma_clean_area SYM_FUNC_END_PI(__dma_map_area) diff --git a/arch/csky/include/asm/processor.h b/arch/csky/include/asm/processor.h index 4800f6563abb..baf9388d5952 100644 --- a/arch/csky/include/asm/processor.h +++ b/arch/csky/include/asm/processor.h @@ -6,9 +6,9 @@ #include #include +#include #include #include -#include #include #include #include diff --git a/arch/microblaze/include/asm/page.h b/arch/microblaze/include/asm/page.h index b13463d39b38..3c837abbebf0 100644 --- a/arch/microblaze/include/asm/page.h +++ b/arch/microblaze/include/asm/page.h @@ -162,7 +162,8 @@ extern int page_is_ram(unsigned long pfn); # define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT) # else /* CONFIG_MMU */ # define ARCH_PFN_OFFSET (memory_start >> PAGE_SHIFT) -# define pfn_valid(pfn) ((pfn) < (max_mapnr + ARCH_PFN_OFFSET)) +# define pfn_valid(pfn) ((pfn) >= ARCH_PFN_OFFSET && \ + (pfn) < (max_mapnr + ARCH_PFN_OFFSET)) # endif /* CONFIG_MMU */ # endif /* __ASSEMBLY__ */ diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index b648dd91999b..8c486248e3dd 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -464,6 +464,7 @@ CONFIG_MMC_CRYPTO=y CONFIG_MMC_SDHCI=y CONFIG_MMC_SDHCI_PLTFM=y CONFIG_LEDS_CLASS_FLASH=y +CONFIG_LEDS_CLASS_MULTICOLOR=y CONFIG_LEDS_TRIGGER_TIMER=y CONFIG_LEDS_TRIGGER_TRANSIENT=y CONFIG_EDAC=y diff --git a/block/blk-mq.c b/block/blk-mq.c index 1d61b7c81e9c..5d520b187555 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1675,8 +1675,7 @@ static bool blk_mq_has_sqsched(struct request_queue *q) */ static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q) { - struct blk_mq_hw_ctx *hctx; - + struct blk_mq_ctx *ctx = blk_mq_get_ctx(q); /* * If the IO scheduler does not respect hardware queues when * dispatching, we just don't bother with multiple HW queues and @@ -1684,8 +1683,8 @@ static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q) * just causes lock contention inside the scheduler and pointless cache * bouncing. */ - hctx = blk_mq_map_queue_type(q, HCTX_TYPE_DEFAULT, - raw_smp_processor_id()); + struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, 0, ctx); + if (!blk_mq_hctx_stopped(hctx)) return hctx; return NULL; diff --git a/block/mq-deadline-main.c b/block/mq-deadline-main.c index f441bf5929be..cdcf72e72d98 100644 --- a/block/mq-deadline-main.c +++ b/block/mq-deadline-main.c @@ -749,6 +749,7 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, per_prio = &dd->per_prio[prio]; if (at_head) { list_add(&rq->queuelist, &per_prio->dispatch); + rq->fifo_time = jiffies; } else { deadline_add_rq_rb(per_prio, rq); @@ -866,7 +867,7 @@ SHOW_JIFFIES(deadline_write_expire_show, dd->fifo_expire[DD_WRITE]); SHOW_JIFFIES(deadline_aging_expire_show, dd->aging_expire); SHOW_INT(deadline_writes_starved_show, dd->writes_starved); SHOW_INT(deadline_front_merges_show, dd->front_merges); -SHOW_INT(deadline_async_depth_show, dd->front_merges); +SHOW_INT(deadline_async_depth_show, dd->async_depth); SHOW_INT(deadline_fifo_batch_show, dd->fifo_batch); #undef SHOW_INT #undef SHOW_JIFFIES @@ -896,7 +897,7 @@ STORE_JIFFIES(deadline_write_expire_store, &dd->fifo_expire[DD_WRITE], 0, INT_MA STORE_JIFFIES(deadline_aging_expire_store, &dd->aging_expire, 0, INT_MAX); STORE_INT(deadline_writes_starved_store, &dd->writes_starved, INT_MIN, INT_MAX); STORE_INT(deadline_front_merges_store, &dd->front_merges, 0, 1); -STORE_INT(deadline_async_depth_store, &dd->front_merges, 1, INT_MAX); +STORE_INT(deadline_async_depth_store, &dd->async_depth, 1, INT_MAX); STORE_INT(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX); #undef STORE_FUNCTION #undef STORE_INT diff --git a/build.config.allmodconfig b/build.config.allmodconfig index a48f8d420208..e5cbe7faa60a 100644 --- a/build.config.allmodconfig +++ b/build.config.allmodconfig @@ -9,6 +9,7 @@ function update_config() { -d CPU_BIG_ENDIAN \ -d DYNAMIC_FTRACE \ -e UNWINDER_FRAME_POINTER \ + -d WERROR \ (cd ${OUT_DIR} && \ make O=${OUT_DIR} $archsubarch CROSS_COMPILE=${CROSS_COMPILE} "${TOOL_ARGS[@]}" ${MAKE_ARGS} olddefconfig) diff --git a/crypto/Makefile b/crypto/Makefile index 2785e5fab9e7..8be22a6ea3d8 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -214,7 +214,7 @@ crypto-fips-objs := drbg.o ecb.o cbc.o ctr.o cts.o gcm.o xts.o hmac.o cmac.o \ gf128mul.o aes_generic.o lib-crypto-aes.o \ jitterentropy.o jitterentropy-kcapi.o \ sha1_generic.o sha256_generic.o sha512_generic.o \ - lib-sha1.o lib-crypto-sha256.o + lib-memneq.o lib-sha1.o lib-crypto-sha256.o crypto-fips-objs := $(foreach o,$(crypto-fips-objs),$(o:.o=-fips.o)) # get the arch to add its objects to $(crypto-fips-objs) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index e2f6da3332a6..d07a6d269776 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -1774,15 +1774,21 @@ static void binder_cleanup_transaction(struct binder_transaction *t, /** * binder_get_object() - gets object and checks for valid metadata * @proc: binder_proc owning the buffer + * @u: sender's user pointer to base of buffer * @buffer: binder_buffer that we're parsing. * @offset: offset in the @buffer at which to validate an object. * @object: struct binder_object to read into * - * Return: If there's a valid metadata object at @offset in @buffer, the + * Copy the binder object at the given offset into @object. If @u is + * provided then the copy is from the sender's buffer. If not, then + * it is copied from the target's @buffer. + * + * Return: If there's a valid metadata object at @offset, the * size of that object. Otherwise, it returns zero. The object * is read into the struct binder_object pointed to by @object. */ static size_t binder_get_object(struct binder_proc *proc, + const void __user *u, struct binder_buffer *buffer, unsigned long offset, struct binder_object *object) @@ -1792,10 +1798,16 @@ static size_t binder_get_object(struct binder_proc *proc, size_t object_size = 0; read_size = min_t(size_t, sizeof(*object), buffer->data_size - offset); - if (offset > buffer->data_size || read_size < sizeof(*hdr) || - binder_alloc_copy_from_buffer(&proc->alloc, object, buffer, - offset, read_size)) + if (offset > buffer->data_size || read_size < sizeof(*hdr)) return 0; + if (u) { + if (copy_from_user(object, u + offset, read_size)) + return 0; + } else { + if (binder_alloc_copy_from_buffer(&proc->alloc, object, buffer, + offset, read_size)) + return 0; + } /* Ok, now see if we read a complete object. */ hdr = &object->hdr; @@ -1868,7 +1880,7 @@ static struct binder_buffer_object *binder_validate_ptr( b, buffer_offset, sizeof(object_offset))) return NULL; - object_size = binder_get_object(proc, b, object_offset, object); + object_size = binder_get_object(proc, NULL, b, object_offset, object); if (!object_size || object->hdr.type != BINDER_TYPE_PTR) return NULL; if (object_offsetp) @@ -1933,7 +1945,8 @@ static bool binder_validate_fixup(struct binder_proc *proc, unsigned long buffer_offset; struct binder_object last_object; struct binder_buffer_object *last_bbo; - size_t object_size = binder_get_object(proc, b, last_obj_offset, + size_t object_size = binder_get_object(proc, NULL, b, + last_obj_offset, &last_object); if (object_size != sizeof(*last_bbo)) return false; @@ -2048,7 +2061,7 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, if (!binder_alloc_copy_from_buffer(&proc->alloc, &object_offset, buffer, buffer_offset, sizeof(object_offset))) - object_size = binder_get_object(proc, buffer, + object_size = binder_get_object(proc, NULL, buffer, object_offset, &object); if (object_size == 0) { pr_err("transaction release %d bad object at offset %lld, size %zd\n", @@ -2389,16 +2402,266 @@ err_fd_not_accepted: return ret; } -static int binder_translate_fd_array(struct binder_fd_array_object *fda, +/** + * struct binder_ptr_fixup - data to be fixed-up in target buffer + * @offset offset in target buffer to fixup + * @skip_size bytes to skip in copy (fixup will be written later) + * @fixup_data data to write at fixup offset + * @node list node + * + * This is used for the pointer fixup list (pf) which is created and consumed + * during binder_transaction() and is only accessed locally. No + * locking is necessary. + * + * The list is ordered by @offset. + */ +struct binder_ptr_fixup { + binder_size_t offset; + size_t skip_size; + binder_uintptr_t fixup_data; + struct list_head node; +}; + +/** + * struct binder_sg_copy - scatter-gather data to be copied + * @offset offset in target buffer + * @sender_uaddr user address in source buffer + * @length bytes to copy + * @node list node + * + * This is used for the sg copy list (sgc) which is created and consumed + * during binder_transaction() and is only accessed locally. No + * locking is necessary. + * + * The list is ordered by @offset. + */ +struct binder_sg_copy { + binder_size_t offset; + const void __user *sender_uaddr; + size_t length; + struct list_head node; +}; + +/** + * binder_do_deferred_txn_copies() - copy and fixup scatter-gather data + * @alloc: binder_alloc associated with @buffer + * @buffer: binder buffer in target process + * @sgc_head: list_head of scatter-gather copy list + * @pf_head: list_head of pointer fixup list + * + * Processes all elements of @sgc_head, applying fixups from @pf_head + * and copying the scatter-gather data from the source process' user + * buffer to the target's buffer. It is expected that the list creation + * and processing all occurs during binder_transaction() so these lists + * are only accessed in local context. + * + * Return: 0=success, else -errno + */ +static int binder_do_deferred_txn_copies(struct binder_alloc *alloc, + struct binder_buffer *buffer, + struct list_head *sgc_head, + struct list_head *pf_head) +{ + int ret = 0; + struct binder_sg_copy *sgc, *tmpsgc; + struct binder_ptr_fixup *tmppf; + struct binder_ptr_fixup *pf = + list_first_entry_or_null(pf_head, struct binder_ptr_fixup, + node); + + list_for_each_entry_safe(sgc, tmpsgc, sgc_head, node) { + size_t bytes_copied = 0; + + while (bytes_copied < sgc->length) { + size_t copy_size; + size_t bytes_left = sgc->length - bytes_copied; + size_t offset = sgc->offset + bytes_copied; + + /* + * We copy up to the fixup (pointed to by pf) + */ + copy_size = pf ? min(bytes_left, (size_t)pf->offset - offset) + : bytes_left; + if (!ret && copy_size) + ret = binder_alloc_copy_user_to_buffer( + alloc, buffer, + offset, + sgc->sender_uaddr + bytes_copied, + copy_size); + bytes_copied += copy_size; + if (copy_size != bytes_left) { + BUG_ON(!pf); + /* we stopped at a fixup offset */ + if (pf->skip_size) { + /* + * we are just skipping. This is for + * BINDER_TYPE_FDA where the translated + * fds will be fixed up when we get + * to target context. + */ + bytes_copied += pf->skip_size; + } else { + /* apply the fixup indicated by pf */ + if (!ret) + ret = binder_alloc_copy_to_buffer( + alloc, buffer, + pf->offset, + &pf->fixup_data, + sizeof(pf->fixup_data)); + bytes_copied += sizeof(pf->fixup_data); + } + list_del(&pf->node); + kfree(pf); + pf = list_first_entry_or_null(pf_head, + struct binder_ptr_fixup, node); + } + } + list_del(&sgc->node); + kfree(sgc); + } + list_for_each_entry_safe(pf, tmppf, pf_head, node) { + BUG_ON(pf->skip_size == 0); + list_del(&pf->node); + kfree(pf); + } + BUG_ON(!list_empty(sgc_head)); + + return ret > 0 ? -EINVAL : ret; +} + +/** + * binder_cleanup_deferred_txn_lists() - free specified lists + * @sgc_head: list_head of scatter-gather copy list + * @pf_head: list_head of pointer fixup list + * + * Called to clean up @sgc_head and @pf_head if there is an + * error. + */ +static void binder_cleanup_deferred_txn_lists(struct list_head *sgc_head, + struct list_head *pf_head) +{ + struct binder_sg_copy *sgc, *tmpsgc; + struct binder_ptr_fixup *pf, *tmppf; + + list_for_each_entry_safe(sgc, tmpsgc, sgc_head, node) { + list_del(&sgc->node); + kfree(sgc); + } + list_for_each_entry_safe(pf, tmppf, pf_head, node) { + list_del(&pf->node); + kfree(pf); + } +} + +/** + * binder_defer_copy() - queue a scatter-gather buffer for copy + * @sgc_head: list_head of scatter-gather copy list + * @offset: binder buffer offset in target process + * @sender_uaddr: user address in source process + * @length: bytes to copy + * + * Specify a scatter-gather block to be copied. The actual copy must + * be deferred until all the needed fixups are identified and queued. + * Then the copy and fixups are done together so un-translated values + * from the source are never visible in the target buffer. + * + * We are guaranteed that repeated calls to this function will have + * monotonically increasing @offset values so the list will naturally + * be ordered. + * + * Return: 0=success, else -errno + */ +static int binder_defer_copy(struct list_head *sgc_head, binder_size_t offset, + const void __user *sender_uaddr, size_t length) +{ + struct binder_sg_copy *bc = kzalloc(sizeof(*bc), GFP_KERNEL); + + if (!bc) + return -ENOMEM; + + bc->offset = offset; + bc->sender_uaddr = sender_uaddr; + bc->length = length; + INIT_LIST_HEAD(&bc->node); + + /* + * We are guaranteed that the deferred copies are in-order + * so just add to the tail. + */ + list_add_tail(&bc->node, sgc_head); + + return 0; +} + +/** + * binder_add_fixup() - queue a fixup to be applied to sg copy + * @pf_head: list_head of binder ptr fixup list + * @offset: binder buffer offset in target process + * @fixup: bytes to be copied for fixup + * @skip_size: bytes to skip when copying (fixup will be applied later) + * + * Add the specified fixup to a list ordered by @offset. When copying + * the scatter-gather buffers, the fixup will be copied instead of + * data from the source buffer. For BINDER_TYPE_FDA fixups, the fixup + * will be applied later (in target process context), so we just skip + * the bytes specified by @skip_size. If @skip_size is 0, we copy the + * value in @fixup. + * + * This function is called *mostly* in @offset order, but there are + * exceptions. Since out-of-order inserts are relatively uncommon, + * we insert the new element by searching backward from the tail of + * the list. + * + * Return: 0=success, else -errno + */ +static int binder_add_fixup(struct list_head *pf_head, binder_size_t offset, + binder_uintptr_t fixup, size_t skip_size) +{ + struct binder_ptr_fixup *pf = kzalloc(sizeof(*pf), GFP_KERNEL); + struct binder_ptr_fixup *tmppf; + + if (!pf) + return -ENOMEM; + + pf->offset = offset; + pf->fixup_data = fixup; + pf->skip_size = skip_size; + INIT_LIST_HEAD(&pf->node); + + /* Fixups are *mostly* added in-order, but there are some + * exceptions. Look backwards through list for insertion point. + */ + list_for_each_entry_reverse(tmppf, pf_head, node) { + if (tmppf->offset < pf->offset) { + list_add(&pf->node, &tmppf->node); + return 0; + } + } + /* + * if we get here, then the new offset is the lowest so + * insert at the head + */ + list_add(&pf->node, pf_head); + return 0; +} + +static int binder_translate_fd_array(struct list_head *pf_head, + struct binder_fd_array_object *fda, + const void __user *sender_ubuffer, struct binder_buffer_object *parent, + struct binder_buffer_object *sender_uparent, struct binder_transaction *t, struct binder_thread *thread, struct binder_transaction *in_reply_to) { binder_size_t fdi, fd_buf_size; binder_size_t fda_offset; + const void __user *sender_ufda_base; struct binder_proc *proc = thread->proc; - struct binder_proc *target_proc = t->to_proc; + int ret; + + if (fda->num_fds == 0) + return 0; fd_buf_size = sizeof(u32) * fda->num_fds; if (fda->num_fds >= SIZE_MAX / sizeof(u32)) { @@ -2422,19 +2685,25 @@ static int binder_translate_fd_array(struct binder_fd_array_object *fda, */ fda_offset = (parent->buffer - (uintptr_t)t->buffer->user_data) + fda->parent_offset; - if (!IS_ALIGNED((unsigned long)fda_offset, sizeof(u32))) { + sender_ufda_base = (void __user *)(uintptr_t)sender_uparent->buffer + + fda->parent_offset; + + if (!IS_ALIGNED((unsigned long)fda_offset, sizeof(u32)) || + !IS_ALIGNED((unsigned long)sender_ufda_base, sizeof(u32))) { binder_user_error("%d:%d parent offset not aligned correctly.\n", proc->pid, thread->pid); return -EINVAL; } + ret = binder_add_fixup(pf_head, fda_offset, 0, fda->num_fds * sizeof(u32)); + if (ret) + return ret; + for (fdi = 0; fdi < fda->num_fds; fdi++) { u32 fd; - int ret; binder_size_t offset = fda_offset + fdi * sizeof(fd); + binder_size_t sender_uoffset = fdi * sizeof(fd); - ret = binder_alloc_copy_from_buffer(&target_proc->alloc, - &fd, t->buffer, - offset, sizeof(fd)); + ret = copy_from_user(&fd, sender_ufda_base + sender_uoffset, sizeof(fd)); if (!ret) ret = binder_translate_fd(fd, offset, t, thread, in_reply_to); @@ -2444,7 +2713,8 @@ static int binder_translate_fd_array(struct binder_fd_array_object *fda, return 0; } -static int binder_fixup_parent(struct binder_transaction *t, +static int binder_fixup_parent(struct list_head *pf_head, + struct binder_transaction *t, struct binder_thread *thread, struct binder_buffer_object *bp, binder_size_t off_start_offset, @@ -2490,14 +2760,7 @@ static int binder_fixup_parent(struct binder_transaction *t, } buffer_offset = bp->parent_offset + (uintptr_t)parent->buffer - (uintptr_t)b->user_data; - if (binder_alloc_copy_to_buffer(&target_proc->alloc, b, buffer_offset, - &bp->buffer, sizeof(bp->buffer))) { - binder_user_error("%d:%d got transaction with invalid parent offset\n", - proc->pid, thread->pid); - return -EINVAL; - } - - return 0; + return binder_add_fixup(pf_head, buffer_offset, bp->buffer, 0); } /** @@ -2715,6 +2978,7 @@ static void binder_transaction(struct binder_proc *proc, binder_size_t off_start_offset, off_end_offset; binder_size_t off_min; binder_size_t sg_buf_offset, sg_buf_end_offset; + binder_size_t user_offset = 0; struct binder_proc *target_proc = NULL; struct binder_thread *target_thread = NULL; struct binder_node *target_node = NULL; @@ -2729,6 +2993,12 @@ static void binder_transaction(struct binder_proc *proc, int t_debug_id = atomic_inc_return(&binder_last_id); char *secctx = NULL; u32 secctx_sz = 0; + struct list_head sgc_head; + struct list_head pf_head; + const void __user *user_buffer = (const void __user *) + (uintptr_t)tr->data.ptr.buffer; + INIT_LIST_HEAD(&sgc_head); + INIT_LIST_HEAD(&pf_head); e = binder_transaction_log_add(&binder_transaction_log); e->debug_id = t_debug_id; @@ -3064,19 +3334,6 @@ static void binder_transaction(struct binder_proc *proc, t->buffer->clear_on_free = !!(t->flags & TF_CLEAR_BUF); trace_binder_transaction_alloc_buf(t->buffer); - if (binder_alloc_copy_user_to_buffer( - &target_proc->alloc, - t->buffer, 0, - (const void __user *) - (uintptr_t)tr->data.ptr.buffer, - tr->data_size)) { - binder_user_error("%d:%d got transaction with invalid data ptr\n", - proc->pid, thread->pid); - return_error = BR_FAILED_REPLY; - return_error_param = -EFAULT; - return_error_line = __LINE__; - goto err_copy_data_failed; - } if (binder_alloc_copy_user_to_buffer( &target_proc->alloc, t->buffer, @@ -3121,6 +3378,7 @@ static void binder_transaction(struct binder_proc *proc, size_t object_size; struct binder_object object; binder_size_t object_offset; + binder_size_t copy_size; if (binder_alloc_copy_from_buffer(&target_proc->alloc, &object_offset, @@ -3132,8 +3390,27 @@ static void binder_transaction(struct binder_proc *proc, return_error_line = __LINE__; goto err_bad_offset; } - object_size = binder_get_object(target_proc, t->buffer, - object_offset, &object); + + /* + * Copy the source user buffer up to the next object + * that will be processed. + */ + copy_size = object_offset - user_offset; + if (copy_size && (user_offset > object_offset || + binder_alloc_copy_user_to_buffer( + &target_proc->alloc, + t->buffer, user_offset, + user_buffer + user_offset, + copy_size))) { + binder_user_error("%d:%d got transaction with invalid data ptr\n", + proc->pid, thread->pid); + return_error = BR_FAILED_REPLY; + return_error_param = -EFAULT; + return_error_line = __LINE__; + goto err_copy_data_failed; + } + object_size = binder_get_object(target_proc, user_buffer, + t->buffer, object_offset, &object); if (object_size == 0 || object_offset < off_min) { binder_user_error("%d:%d got transaction with invalid offset (%lld, min %lld max %lld) or object.\n", proc->pid, thread->pid, @@ -3145,6 +3422,11 @@ static void binder_transaction(struct binder_proc *proc, return_error_line = __LINE__; goto err_bad_offset; } + /* + * Set offset to the next buffer fragment to be + * copied + */ + user_offset = object_offset + object_size; hdr = &object.hdr; off_min = object_offset + object_size; @@ -3207,6 +3489,8 @@ static void binder_transaction(struct binder_proc *proc, case BINDER_TYPE_FDA: { struct binder_object ptr_object; binder_size_t parent_offset; + struct binder_object user_object; + size_t user_parent_size; struct binder_fd_array_object *fda = to_binder_fd_array_object(hdr); size_t num_valid = (buffer_offset - off_start_offset) / @@ -3238,11 +3522,35 @@ static void binder_transaction(struct binder_proc *proc, return_error_line = __LINE__; goto err_bad_parent; } - ret = binder_translate_fd_array(fda, parent, t, thread, - in_reply_to); - if (ret < 0) { + /* + * We need to read the user version of the parent + * object to get the original user offset + */ + user_parent_size = + binder_get_object(proc, user_buffer, t->buffer, + parent_offset, &user_object); + if (user_parent_size != sizeof(user_object.bbo)) { + binder_user_error("%d:%d invalid ptr object size: %zd vs %zd\n", + proc->pid, thread->pid, + user_parent_size, + sizeof(user_object.bbo)); return_error = BR_FAILED_REPLY; - return_error_param = ret; + return_error_param = -EINVAL; + return_error_line = __LINE__; + goto err_bad_parent; + } + ret = binder_translate_fd_array(&pf_head, fda, + user_buffer, parent, + &user_object.bbo, t, + thread, in_reply_to); + if (!ret) + ret = binder_alloc_copy_to_buffer(&target_proc->alloc, + t->buffer, + object_offset, + fda, sizeof(*fda)); + if (ret) { + return_error = BR_FAILED_REPLY; + return_error_param = ret > 0 ? -EINVAL : ret; return_error_line = __LINE__; goto err_translate_failed; } @@ -3264,19 +3572,14 @@ static void binder_transaction(struct binder_proc *proc, return_error_line = __LINE__; goto err_bad_offset; } - if (binder_alloc_copy_user_to_buffer( - &target_proc->alloc, - t->buffer, - sg_buf_offset, - (const void __user *) - (uintptr_t)bp->buffer, - bp->length)) { - binder_user_error("%d:%d got transaction with invalid offsets ptr\n", - proc->pid, thread->pid); - return_error_param = -EFAULT; + ret = binder_defer_copy(&sgc_head, sg_buf_offset, + (const void __user *)(uintptr_t)bp->buffer, + bp->length); + if (ret) { return_error = BR_FAILED_REPLY; + return_error_param = ret; return_error_line = __LINE__; - goto err_copy_data_failed; + goto err_translate_failed; } /* Fixup buffer pointer to target proc address space */ bp->buffer = (uintptr_t) @@ -3285,7 +3588,8 @@ static void binder_transaction(struct binder_proc *proc, num_valid = (buffer_offset - off_start_offset) / sizeof(binder_size_t); - ret = binder_fixup_parent(t, thread, bp, + ret = binder_fixup_parent(&pf_head, t, + thread, bp, off_start_offset, num_valid, last_fixup_obj_off, @@ -3312,6 +3616,30 @@ static void binder_transaction(struct binder_proc *proc, goto err_bad_object_type; } } + /* Done processing objects, copy the rest of the buffer */ + if (binder_alloc_copy_user_to_buffer( + &target_proc->alloc, + t->buffer, user_offset, + user_buffer + user_offset, + tr->data_size - user_offset)) { + binder_user_error("%d:%d got transaction with invalid data ptr\n", + proc->pid, thread->pid); + return_error = BR_FAILED_REPLY; + return_error_param = -EFAULT; + return_error_line = __LINE__; + goto err_copy_data_failed; + } + + ret = binder_do_deferred_txn_copies(&target_proc->alloc, t->buffer, + &sgc_head, &pf_head); + if (ret) { + binder_user_error("%d:%d got transaction with invalid offsets ptr\n", + proc->pid, thread->pid); + return_error = BR_FAILED_REPLY; + return_error_param = ret; + return_error_line = __LINE__; + goto err_copy_data_failed; + } if (t->buffer->oneway_spam_suspect) tcomplete->type = BINDER_WORK_TRANSACTION_ONEWAY_SPAM_SUSPECT; else @@ -3387,6 +3715,7 @@ err_bad_object_type: err_bad_offset: err_bad_parent: err_copy_data_failed: + binder_cleanup_deferred_txn_lists(&sgc_head, &pf_head); binder_free_txn_fixups(t); trace_binder_transaction_failed_buffer_release(t->buffer); binder_transaction_buffer_release(target_proc, NULL, t->buffer, diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 7ac6fd6d69c8..3ff54d2aa04d 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -7,6 +7,7 @@ */ #ifndef __GENKSYMS__ +#include #include #endif @@ -74,6 +75,7 @@ #include #include #include +#include /* * Export tracepoints that act as a bare tracehook (ie: have no trace event @@ -462,12 +464,15 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_alloc_si); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_si); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_free_pages); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_shmem_page_flag); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mmput); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sched_pelt_multiplier); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_pages_reclaim_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_pages_failure_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_page_look_around_ref); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_look_around); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_look_around_migrate_page); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_test_clear_look_around_ref); - +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_dma_buf_stats_teardown); /* * For type visibility */ diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index cbe4c6f524bb..c4e56e058599 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -219,6 +219,9 @@ static void genpd_debug_remove(struct generic_pm_domain *genpd) { struct dentry *d; + if (!genpd_debugfs_dir) + return; + d = debugfs_lookup(genpd->name, genpd_debugfs_dir); debugfs_remove(d); } @@ -1972,6 +1975,7 @@ int pm_genpd_init(struct generic_pm_domain *genpd, genpd->device_count = 0; genpd->max_off_time_ns = -1; genpd->max_off_time_changed = true; + genpd->next_wakeup = KTIME_MAX; genpd->provider = NULL; genpd->has_provider = false; genpd->accounting_time = ktime_get(); @@ -2052,9 +2056,9 @@ static int genpd_remove(struct generic_pm_domain *genpd) kfree(link); } - genpd_debug_remove(genpd); list_del(&genpd->gpd_list_node); genpd_unlock(genpd); + genpd_debug_remove(genpd); cancel_work_sync(&genpd->power_off_work); if (genpd_is_cpu_domain(genpd)) free_cpumask_var(genpd->cpus); diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index ba791175317f..2529318d2747 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -3827,8 +3827,9 @@ struct clk *clk_hw_create_clk(struct device *dev, struct clk_hw *hw, struct clk *clk_hw_get_clk(struct clk_hw *hw, const char *con_id) { struct device *dev = hw->core->dev; + const char *name = dev ? dev_name(dev) : NULL; - return clk_hw_create_clk(dev, hw, dev_name(dev), con_id); + return clk_hw_create_clk(dev, hw, name, con_id); } EXPORT_SYMBOL(clk_hw_get_clk); diff --git a/drivers/clk/versatile/Kconfig b/drivers/clk/versatile/Kconfig index 792315d893db..481de5657d85 100644 --- a/drivers/clk/versatile/Kconfig +++ b/drivers/clk/versatile/Kconfig @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only menu "Clock driver for ARM Reference designs" + depends on HAS_IOMEM config ICST bool "Clock driver for ARM Reference designs ICST" diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig index 8634c09036ac..3280ca972502 100644 --- a/drivers/dma-buf/Kconfig +++ b/drivers/dma-buf/Kconfig @@ -67,7 +67,7 @@ menuconfig DMABUF_HEAPS menuconfig DMABUF_SYSFS_STATS bool "DMA-BUF sysfs statistics" - select DMA_SHARED_BUFFER + depends on DMA_SHARED_BUFFER help Choose this option to enable DMA-BUF sysfs statistics in location /sys/kernel/dmabuf/buffers. diff --git a/drivers/dma-buf/dma-buf-sysfs-stats.c b/drivers/dma-buf/dma-buf-sysfs-stats.c index 3c7bb2baf8da..5c8efa55e3aa 100644 --- a/drivers/dma-buf/dma-buf-sysfs-stats.c +++ b/drivers/dma-buf/dma-buf-sysfs-stats.c @@ -13,6 +13,8 @@ #include #include +#include + #include "dma-buf-sysfs-stats.h" #define to_dma_buf_entry_from_kobj(x) container_of(x, struct dma_buf_sysfs_entry, kobj) @@ -88,13 +90,17 @@ static struct kobj_type dma_buf_ktype = { void dma_buf_stats_teardown(struct dma_buf *dmabuf) { struct dma_buf_sysfs_entry *sysfs_entry; + bool skip_sysfs_release = false; sysfs_entry = dmabuf->sysfs_entry; if (!sysfs_entry) return; - kobject_del(&sysfs_entry->kobj); - kobject_put(&sysfs_entry->kobj); + trace_android_rvh_dma_buf_stats_teardown(sysfs_entry, &skip_sysfs_release); + if (!skip_sysfs_release) { + kobject_del(&sysfs_entry->kobj); + kobject_put(&sysfs_entry->kobj); + } } /* @@ -136,15 +142,21 @@ void dma_buf_uninit_sysfs_statistics(void) kset_unregister(dma_buf_stats_kset); } +struct dma_buf_create_sysfs_entry { + struct dma_buf *dmabuf; + struct work_struct work; +}; + +union dma_buf_create_sysfs_work_entry { + struct dma_buf_create_sysfs_entry create_entry; + struct dma_buf_sysfs_entry sysfs_entry; +}; + static void sysfs_add_workfn(struct work_struct *work) { - /* The ABI would have to change for this to be false, but let's be paranoid. */ - _Static_assert(sizeof(struct kobject) >= sizeof(struct work_struct), - "kobject is smaller than work_struct"); - - struct dma_buf_sysfs_entry *sysfs_entry = - container_of((struct kobject *)work, struct dma_buf_sysfs_entry, kobj); - struct dma_buf *dmabuf = sysfs_entry->dmabuf; + struct dma_buf_create_sysfs_entry *create_entry = + container_of(work, struct dma_buf_create_sysfs_entry, work); + struct dma_buf *dmabuf = create_entry->dmabuf; /* * A dmabuf is ref-counted via its file member. If this handler holds the only @@ -155,6 +167,7 @@ static void sysfs_add_workfn(struct work_struct *work) * is released, and that can't happen until the end of this function. */ if (file_count(dmabuf->file) > 1) { + dmabuf->sysfs_entry->dmabuf = dmabuf; /* * kobject_init_and_add expects kobject to be zero-filled, but we have populated it * to trigger this work function. @@ -179,8 +192,8 @@ static void sysfs_add_workfn(struct work_struct *work) int dma_buf_stats_setup(struct dma_buf *dmabuf) { - struct dma_buf_sysfs_entry *sysfs_entry; - struct work_struct *work; + struct dma_buf_create_sysfs_entry *create_entry; + union dma_buf_create_sysfs_work_entry *work_entry; if (!dmabuf || !dmabuf->file) return -EINVAL; @@ -190,21 +203,18 @@ int dma_buf_stats_setup(struct dma_buf *dmabuf) return -EINVAL; } - sysfs_entry = kmalloc(sizeof(struct dma_buf_sysfs_entry), GFP_KERNEL); - if (!sysfs_entry) + work_entry = kmalloc(sizeof(union dma_buf_create_sysfs_work_entry), GFP_KERNEL); + if (!work_entry) return -ENOMEM; - sysfs_entry->dmabuf = dmabuf; - dmabuf->sysfs_entry = sysfs_entry; + dmabuf->sysfs_entry = &work_entry->sysfs_entry; - /* - * The use of kobj as a work_struct is an ugly hack - * to avoid an ABI break in this frozen kernel. - */ - work = (struct work_struct *)&dmabuf->sysfs_entry->kobj; - INIT_WORK(work, sysfs_add_workfn); + create_entry = &work_entry->create_entry; + create_entry->dmabuf = dmabuf; + + INIT_WORK(&create_entry->work, sysfs_add_workfn); get_dma_buf(dmabuf); /* This reference will be dropped in sysfs_add_workfn. */ - schedule_work(work); + schedule_work(&create_entry->work); return 0; } diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c index fd564aa70ee9..511d678ecbfc 100644 --- a/drivers/dma-buf/heaps/cma_heap.c +++ b/drivers/dma-buf/heaps/cma_heap.c @@ -126,10 +126,11 @@ static int cma_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, struct cma_heap_buffer *buffer = dmabuf->priv; struct dma_heap_attachment *a; + mutex_lock(&buffer->lock); + if (buffer->vmap_cnt) invalidate_kernel_vmap_range(buffer->vaddr, buffer->len); - mutex_lock(&buffer->lock); list_for_each_entry(a, &buffer->attachments, list) { if (!a->mapped) continue; @@ -146,10 +147,11 @@ static int cma_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf, struct cma_heap_buffer *buffer = dmabuf->priv; struct dma_heap_attachment *a; + mutex_lock(&buffer->lock); + if (buffer->vmap_cnt) flush_kernel_vmap_range(buffer->vaddr, buffer->len); - mutex_lock(&buffer->lock); list_for_each_entry(a, &buffer->attachments, list) { if (!a->mapped) continue; diff --git a/drivers/dma-buf/heaps/page_pool.c b/drivers/dma-buf/heaps/page_pool.c index b79e737bac95..de9d72809412 100644 --- a/drivers/dma-buf/heaps/page_pool.c +++ b/drivers/dma-buf/heaps/page_pool.c @@ -11,10 +11,16 @@ #include #include #include +#include #include #include #include "page_pool.h" +struct dmabuf_page_pool_with_spinlock { + struct dmabuf_page_pool pool; + struct spinlock spinlock; +}; + static LIST_HEAD(pool_list); static DEFINE_MUTEX(pool_list_lock); @@ -35,34 +41,41 @@ static inline void dmabuf_page_pool_free_pages(struct dmabuf_page_pool *pool, static void dmabuf_page_pool_add(struct dmabuf_page_pool *pool, struct page *page) { int index; + struct dmabuf_page_pool_with_spinlock *container_pool = + container_of(pool, struct dmabuf_page_pool_with_spinlock, pool); if (PageHighMem(page)) index = POOL_HIGHPAGE; else index = POOL_LOWPAGE; - mutex_lock(&pool->mutex); + spin_lock(&container_pool->spinlock); list_add_tail(&page->lru, &pool->items[index]); pool->count[index]++; + spin_unlock(&container_pool->spinlock); mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, 1 << pool->order); - mutex_unlock(&pool->mutex); } static struct page *dmabuf_page_pool_remove(struct dmabuf_page_pool *pool, int index) { struct page *page; + struct dmabuf_page_pool_with_spinlock *container_pool = + container_of(pool, struct dmabuf_page_pool_with_spinlock, pool); - mutex_lock(&pool->mutex); + spin_lock(&container_pool->spinlock); page = list_first_entry_or_null(&pool->items[index], struct page, lru); if (page) { pool->count[index]--; list_del(&page->lru); + spin_unlock(&container_pool->spinlock); mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, -(1 << pool->order)); + goto out; } - mutex_unlock(&pool->mutex); + spin_unlock(&container_pool->spinlock); +out: return page; } @@ -113,19 +126,25 @@ static int dmabuf_page_pool_total(struct dmabuf_page_pool *pool, bool high) struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask, unsigned int order) { - struct dmabuf_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL); + struct dmabuf_page_pool *pool; + struct dmabuf_page_pool_with_spinlock *container_pool = + kmalloc(sizeof(*container_pool), GFP_KERNEL); int i; - if (!pool) + if (!container_pool) return NULL; + spin_lock_init(&container_pool->spinlock); + pool = &container_pool->pool; + for (i = 0; i < POOL_TYPE_SIZE; i++) { pool->count[i] = 0; INIT_LIST_HEAD(&pool->items[i]); } pool->gfp_mask = gfp_mask | __GFP_COMP; pool->order = order; - mutex_init(&pool->mutex); + mutex_init(&pool->mutex); /* No longer used! */ + mutex_lock(&pool->mutex); /* Make sure anyone who attempts to acquire this hangs */ mutex_lock(&pool_list_lock); list_add(&pool->list, &pool_list); @@ -138,6 +157,7 @@ EXPORT_SYMBOL_GPL(dmabuf_page_pool_create); void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool) { struct page *page; + struct dmabuf_page_pool_with_spinlock *container_pool; int i; /* Remove us from the pool list */ @@ -151,7 +171,8 @@ void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool) dmabuf_page_pool_free_pages(pool, page); } - kfree(pool); + container_pool = container_of(pool, struct dmabuf_page_pool_with_spinlock, pool); + kfree(container_pool); } EXPORT_SYMBOL_GPL(dmabuf_page_pool_destroy); diff --git a/drivers/dma-buf/heaps/page_pool.h b/drivers/dma-buf/heaps/page_pool.h index 6b083b04f195..b578e18dc1ed 100644 --- a/drivers/dma-buf/heaps/page_pool.h +++ b/drivers/dma-buf/heaps/page_pool.h @@ -40,7 +40,7 @@ enum { struct dmabuf_page_pool { int count[POOL_TYPE_SIZE]; struct list_head items[POOL_TYPE_SIZE]; - struct mutex mutex; + struct mutex mutex; /* No longer used! */ gfp_t gfp_mask; unsigned int order; struct list_head list; diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index a5f926825326..2b7f6ea26761 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -1090,6 +1090,7 @@ #define USB_DEVICE_ID_SONY_PS4_CONTROLLER_2 0x09cc #define USB_DEVICE_ID_SONY_PS4_CONTROLLER_DONGLE 0x0ba0 #define USB_DEVICE_ID_SONY_PS5_CONTROLLER 0x0ce6 +#define USB_DEVICE_ID_SONY_PS5_CONTROLLER_2 0x0df2 #define USB_DEVICE_ID_SONY_MOTION_CONTROLLER 0x03d5 #define USB_DEVICE_ID_SONY_NAVIGATION_CONTROLLER 0x042f #define USB_DEVICE_ID_SONY_BUZZ_CONTROLLER 0x0002 diff --git a/drivers/hid/hid-playstation.c b/drivers/hid/hid-playstation.c index ab7c82c2e886..0b58763bfd30 100644 --- a/drivers/hid/hid-playstation.c +++ b/drivers/hid/hid-playstation.c @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include @@ -38,11 +40,13 @@ struct ps_device { uint8_t battery_capacity; int battery_status; + const char *input_dev_name; /* Name of primary input device. */ uint8_t mac_address[6]; /* Note: stored in little endian order. */ uint32_t hw_version; uint32_t fw_version; int (*parse_report)(struct ps_device *dev, struct hid_report *report, u8 *data, int size); + void (*remove)(struct ps_device *dev); }; /* Calibration data for playstation motion sensors. */ @@ -53,6 +57,13 @@ struct ps_calibration_data { int sens_denom; }; +struct ps_led_info { + const char *name; + const char *color; + enum led_brightness (*brightness_get)(struct led_classdev *cdev); + int (*brightness_set)(struct led_classdev *cdev, enum led_brightness); +}; + /* Seed values for DualShock4 / DualSense CRC32 for different report types. */ #define PS_INPUT_CRC32_SEED 0xA1 #define PS_OUTPUT_CRC32_SEED 0xA2 @@ -97,6 +108,9 @@ struct ps_calibration_data { #define DS_STATUS_CHARGING GENMASK(7, 4) #define DS_STATUS_CHARGING_SHIFT 4 +/* Feature version from DualSense Firmware Info report. */ +#define DS_FEATURE_VERSION(major, minor) ((major & 0xff) << 8 | (minor & 0xff)) + /* * Status of a DualSense touch point contact. * Contact IDs, with highest bit set are 'inactive' @@ -115,6 +129,7 @@ struct ps_calibration_data { #define DS_OUTPUT_VALID_FLAG1_RELEASE_LEDS BIT(3) #define DS_OUTPUT_VALID_FLAG1_PLAYER_INDICATOR_CONTROL_ENABLE BIT(4) #define DS_OUTPUT_VALID_FLAG2_LIGHTBAR_SETUP_CONTROL_ENABLE BIT(1) +#define DS_OUTPUT_VALID_FLAG2_COMPATIBLE_VIBRATION2 BIT(2) #define DS_OUTPUT_POWER_SAVE_CONTROL_MIC_MUTE BIT(4) #define DS_OUTPUT_LIGHTBAR_SETUP_LIGHT_OUT BIT(1) @@ -132,6 +147,9 @@ struct dualsense { struct input_dev *sensors; struct input_dev *touchpad; + /* Update version is used as a feature/capability version. */ + uint16_t update_version; + /* Calibration data for accelerometer and gyroscope. */ struct ps_calibration_data accel_calib_data[3]; struct ps_calibration_data gyro_calib_data[3]; @@ -142,11 +160,13 @@ struct dualsense { uint32_t sensor_timestamp_us; /* Compatible rumble state */ + bool use_vibration_v2; bool update_rumble; uint8_t motor_left; uint8_t motor_right; /* RGB lightbar */ + struct led_classdev_mc lightbar; bool update_lightbar; uint8_t lightbar_red; uint8_t lightbar_green; @@ -163,6 +183,7 @@ struct dualsense { struct led_classdev player_leds[5]; struct work_struct output_worker; + bool output_worker_initialized; void *output_report_dmabuf; uint8_t output_seq; /* Sequence number for output report. */ }; @@ -288,6 +309,9 @@ static const struct {int x; int y; } ps_gamepad_hat_mapping[] = { {0, 0}, }; +static inline void dualsense_schedule_work(struct dualsense *ds); +static void dualsense_set_lightbar(struct dualsense *ds, uint8_t red, uint8_t green, uint8_t blue); + /* * Add a new ps_device to ps_devices if it doesn't exist. * Return error on duplicate device, which can happen if the same @@ -525,6 +549,71 @@ static int ps_get_report(struct hid_device *hdev, uint8_t report_id, uint8_t *bu return 0; } +static int ps_led_register(struct ps_device *ps_dev, struct led_classdev *led, + const struct ps_led_info *led_info) +{ + int ret; + + led->name = devm_kasprintf(&ps_dev->hdev->dev, GFP_KERNEL, + "%s:%s:%s", ps_dev->input_dev_name, led_info->color, led_info->name); + + if (!led->name) + return -ENOMEM; + + led->brightness = 0; + led->max_brightness = 1; + led->flags = LED_CORE_SUSPENDRESUME; + led->brightness_get = led_info->brightness_get; + led->brightness_set_blocking = led_info->brightness_set; + + ret = devm_led_classdev_register(&ps_dev->hdev->dev, led); + if (ret) { + hid_err(ps_dev->hdev, "Failed to register LED %s: %d\n", led_info->name, ret); + return ret; + } + + return 0; +} + +/* Register a DualSense/DualShock4 RGB lightbar represented by a multicolor LED. */ +static int ps_lightbar_register(struct ps_device *ps_dev, struct led_classdev_mc *lightbar_mc_dev, + int (*brightness_set)(struct led_classdev *, enum led_brightness)) +{ + struct hid_device *hdev = ps_dev->hdev; + struct mc_subled *mc_led_info; + struct led_classdev *led_cdev; + int ret; + + mc_led_info = devm_kmalloc_array(&hdev->dev, 3, sizeof(*mc_led_info), + GFP_KERNEL | __GFP_ZERO); + if (!mc_led_info) + return -ENOMEM; + + mc_led_info[0].color_index = LED_COLOR_ID_RED; + mc_led_info[1].color_index = LED_COLOR_ID_GREEN; + mc_led_info[2].color_index = LED_COLOR_ID_BLUE; + + lightbar_mc_dev->subled_info = mc_led_info; + lightbar_mc_dev->num_colors = 3; + + led_cdev = &lightbar_mc_dev->led_cdev; + led_cdev->name = devm_kasprintf(&hdev->dev, GFP_KERNEL, "%s:rgb:indicator", + ps_dev->input_dev_name); + if (!led_cdev->name) + return -ENOMEM; + led_cdev->brightness = 255; + led_cdev->max_brightness = 255; + led_cdev->brightness_set_blocking = brightness_set; + + ret = devm_led_classdev_multicolor_register(&hdev->dev, lightbar_mc_dev); + if (ret < 0) { + hid_err(hdev, "Cannot register multicolor LED device\n"); + return ret; + } + + return 0; +} + static struct input_dev *ps_sensors_create(struct hid_device *hdev, int accel_range, int accel_res, int gyro_range, int gyro_res) { @@ -614,15 +703,12 @@ static ssize_t hardware_version_show(struct device *dev, static DEVICE_ATTR_RO(hardware_version); -static struct attribute *ps_device_attributes[] = { +static struct attribute *ps_device_attrs[] = { &dev_attr_firmware_version.attr, &dev_attr_hardware_version.attr, NULL }; - -static const struct attribute_group ps_device_attribute_group = { - .attrs = ps_device_attributes, -}; +ATTRIBUTE_GROUPS(ps_device); static int dualsense_get_calibration_data(struct dualsense *ds) { @@ -714,6 +800,7 @@ err_free: return ret; } + static int dualsense_get_firmware_info(struct dualsense *ds) { uint8_t *buf; @@ -733,6 +820,15 @@ static int dualsense_get_firmware_info(struct dualsense *ds) ds->base.hw_version = get_unaligned_le32(&buf[24]); ds->base.fw_version = get_unaligned_le32(&buf[28]); + /* Update version is some kind of feature version. It is distinct from + * the firmware version as there can be many different variations of a + * controller over time with the same physical shell, but with different + * PCBs and other internal changes. The update version (internal name) is + * used as a means to detect what features are available and change behavior. + * Note: the version is different between DualSense and DualSense Edge. + */ + ds->update_version = get_unaligned_le16(&buf[44]); + err_free: kfree(buf); return ret; @@ -761,6 +857,53 @@ err_free: return ret; } +static int dualsense_lightbar_set_brightness(struct led_classdev *cdev, + enum led_brightness brightness) +{ + struct led_classdev_mc *mc_cdev = lcdev_to_mccdev(cdev); + struct dualsense *ds = container_of(mc_cdev, struct dualsense, lightbar); + uint8_t red, green, blue; + + led_mc_calc_color_components(mc_cdev, brightness); + red = mc_cdev->subled_info[0].brightness; + green = mc_cdev->subled_info[1].brightness; + blue = mc_cdev->subled_info[2].brightness; + + dualsense_set_lightbar(ds, red, green, blue); + return 0; +} + +static enum led_brightness dualsense_player_led_get_brightness(struct led_classdev *led) +{ + struct hid_device *hdev = to_hid_device(led->dev->parent); + struct dualsense *ds = hid_get_drvdata(hdev); + + return !!(ds->player_leds_state & BIT(led - ds->player_leds)); +} + +static int dualsense_player_led_set_brightness(struct led_classdev *led, enum led_brightness value) +{ + struct hid_device *hdev = to_hid_device(led->dev->parent); + struct dualsense *ds = hid_get_drvdata(hdev); + unsigned long flags; + unsigned int led_index; + + spin_lock_irqsave(&ds->base.lock, flags); + + led_index = led - ds->player_leds; + if (value == LED_OFF) + ds->player_leds_state &= ~BIT(led_index); + else + ds->player_leds_state |= BIT(led_index); + + ds->update_player_leds = true; + spin_unlock_irqrestore(&ds->base.lock, flags); + + dualsense_schedule_work(ds); + + return 0; +} + static void dualsense_init_output_report(struct dualsense *ds, struct dualsense_output_report *rp, void *buf) { @@ -800,6 +943,16 @@ static void dualsense_init_output_report(struct dualsense *ds, struct dualsense_ } } +static inline void dualsense_schedule_work(struct dualsense *ds) +{ + unsigned long flags; + + spin_lock_irqsave(&ds->base.lock, flags); + if (ds->output_worker_initialized) + schedule_work(&ds->output_worker); + spin_unlock_irqrestore(&ds->base.lock, flags); +} + /* * Helper function to send DualSense output reports. Applies a CRC at the end of a report * for Bluetooth reports. @@ -838,7 +991,10 @@ static void dualsense_output_worker(struct work_struct *work) if (ds->update_rumble) { /* Select classic rumble style haptics and enable it. */ common->valid_flag0 |= DS_OUTPUT_VALID_FLAG0_HAPTICS_SELECT; - common->valid_flag0 |= DS_OUTPUT_VALID_FLAG0_COMPATIBLE_VIBRATION; + if (ds->use_vibration_v2) + common->valid_flag2 |= DS_OUTPUT_VALID_FLAG2_COMPATIBLE_VIBRATION2; + else + common->valid_flag0 |= DS_OUTPUT_VALID_FLAG0_COMPATIBLE_VIBRATION; common->motor_left = ds->motor_left; common->motor_right = ds->motor_right; ds->update_rumble = false; @@ -960,7 +1116,7 @@ static int dualsense_parse_report(struct ps_device *ps_dev, struct hid_report *r spin_unlock_irqrestore(&ps_dev->lock, flags); /* Schedule updating of microphone state at hardware level. */ - schedule_work(&ds->output_worker); + dualsense_schedule_work(ds); } ds->last_btn_mic_state = btn_mic_state; @@ -1075,10 +1231,22 @@ static int dualsense_play_effect(struct input_dev *dev, void *data, struct ff_ef ds->motor_right = effect->u.rumble.weak_magnitude / 256; spin_unlock_irqrestore(&ds->base.lock, flags); - schedule_work(&ds->output_worker); + dualsense_schedule_work(ds); return 0; } +static void dualsense_remove(struct ps_device *ps_dev) +{ + struct dualsense *ds = container_of(ps_dev, struct dualsense, base); + unsigned long flags; + + spin_lock_irqsave(&ds->base.lock, flags); + ds->output_worker_initialized = false; + spin_unlock_irqrestore(&ds->base.lock, flags); + + cancel_work_sync(&ds->output_worker); +} + static int dualsense_reset_leds(struct dualsense *ds) { struct dualsense_output_report report; @@ -1106,12 +1274,16 @@ static int dualsense_reset_leds(struct dualsense *ds) static void dualsense_set_lightbar(struct dualsense *ds, uint8_t red, uint8_t green, uint8_t blue) { + unsigned long flags; + + spin_lock_irqsave(&ds->base.lock, flags); ds->update_lightbar = true; ds->lightbar_red = red; ds->lightbar_green = green; ds->lightbar_blue = blue; + spin_unlock_irqrestore(&ds->base.lock, flags); - schedule_work(&ds->output_worker); + dualsense_schedule_work(ds); } static void dualsense_set_player_leds(struct dualsense *ds) @@ -1134,7 +1306,7 @@ static void dualsense_set_player_leds(struct dualsense *ds) ds->update_player_leds = true; ds->player_leds_state = player_ids[player_id]; - schedule_work(&ds->output_worker); + dualsense_schedule_work(ds); } static struct ps_device *dualsense_create(struct hid_device *hdev) @@ -1142,7 +1314,20 @@ static struct ps_device *dualsense_create(struct hid_device *hdev) struct dualsense *ds; struct ps_device *ps_dev; uint8_t max_output_report_size; - int ret; + int i, ret; + + static const struct ps_led_info player_leds_info[] = { + { LED_FUNCTION_PLAYER1, "white", dualsense_player_led_get_brightness, + dualsense_player_led_set_brightness }, + { LED_FUNCTION_PLAYER2, "white", dualsense_player_led_get_brightness, + dualsense_player_led_set_brightness }, + { LED_FUNCTION_PLAYER3, "white", dualsense_player_led_get_brightness, + dualsense_player_led_set_brightness }, + { LED_FUNCTION_PLAYER4, "white", dualsense_player_led_get_brightness, + dualsense_player_led_set_brightness }, + { LED_FUNCTION_PLAYER5, "white", dualsense_player_led_get_brightness, + dualsense_player_led_set_brightness } + }; ds = devm_kzalloc(&hdev->dev, sizeof(*ds), GFP_KERNEL); if (!ds) @@ -1160,7 +1345,9 @@ static struct ps_device *dualsense_create(struct hid_device *hdev) ps_dev->battery_capacity = 100; /* initial value until parse_report. */ ps_dev->battery_status = POWER_SUPPLY_STATUS_UNKNOWN; ps_dev->parse_report = dualsense_parse_report; + ps_dev->remove = dualsense_remove; INIT_WORK(&ds->output_worker, dualsense_output_worker); + ds->output_worker_initialized = true; hid_set_drvdata(hdev, ds); max_output_report_size = sizeof(struct dualsense_output_report_bt); @@ -1181,6 +1368,21 @@ static struct ps_device *dualsense_create(struct hid_device *hdev) return ERR_PTR(ret); } + /* Original DualSense firmware simulated classic controller rumble through + * its new haptics hardware. It felt different from classic rumble users + * were used to. Since then new firmwares were introduced to change behavior + * and make this new 'v2' behavior default on PlayStation and other platforms. + * The original DualSense requires a new enough firmware as bundled with PS5 + * software released in 2021. DualSense edge supports it out of the box. + * Both devices also support the old mode, but it is not really used. + */ + if (hdev->product == USB_DEVICE_ID_SONY_PS5_CONTROLLER) { + /* Feature version 2.21 introduced new vibration method. */ + ds->use_vibration_v2 = ds->update_version >= DS_FEATURE_VERSION(2, 21); + } else if (hdev->product == USB_DEVICE_ID_SONY_PS5_CONTROLLER_2) { + ds->use_vibration_v2 = true; + } + ret = ps_devices_list_add(ps_dev); if (ret) return ERR_PTR(ret); @@ -1196,6 +1398,8 @@ static struct ps_device *dualsense_create(struct hid_device *hdev) ret = PTR_ERR(ds->gamepad); goto err; } + /* Use gamepad input device name as primary device name for e.g. LEDs */ + ps_dev->input_dev_name = dev_name(&ds->gamepad->dev); ds->sensors = ps_sensors_create(hdev, DS_ACC_RANGE, DS_ACC_RES_PER_G, DS_GYRO_RANGE, DS_GYRO_RES_PER_DEG_S); @@ -1223,8 +1427,21 @@ static struct ps_device *dualsense_create(struct hid_device *hdev) if (ret) goto err; + ret = ps_lightbar_register(ps_dev, &ds->lightbar, dualsense_lightbar_set_brightness); + if (ret) + goto err; + + /* Set default lightbar color. */ dualsense_set_lightbar(ds, 0, 0, 128); /* blue */ + for (i = 0; i < ARRAY_SIZE(player_leds_info); i++) { + const struct ps_led_info *led_info = &player_leds_info[i]; + + ret = ps_led_register(ps_dev, &ds->player_leds[i], led_info); + if (ret < 0) + goto err; + } + ret = ps_device_set_player_id(ps_dev); if (ret) { hid_err(hdev, "Failed to assign player id for DualSense: %d\n", ret); @@ -1282,7 +1499,8 @@ static int ps_probe(struct hid_device *hdev, const struct hid_device_id *id) goto err_stop; } - if (hdev->product == USB_DEVICE_ID_SONY_PS5_CONTROLLER) { + if (hdev->product == USB_DEVICE_ID_SONY_PS5_CONTROLLER || + hdev->product == USB_DEVICE_ID_SONY_PS5_CONTROLLER_2) { dev = dualsense_create(hdev); if (IS_ERR(dev)) { hid_err(hdev, "Failed to create dualsense.\n"); @@ -1291,12 +1509,6 @@ static int ps_probe(struct hid_device *hdev, const struct hid_device_id *id) } } - ret = devm_device_add_group(&hdev->dev, &ps_device_attribute_group); - if (ret) { - hid_err(hdev, "Failed to register sysfs nodes.\n"); - goto err_close; - } - return ret; err_close: @@ -1313,6 +1525,9 @@ static void ps_remove(struct hid_device *hdev) ps_devices_list_remove(dev); ps_device_release_player_id(dev); + if (dev->remove) + dev->remove(dev); + hid_hw_close(hdev); hid_hw_stop(hdev); } @@ -1320,6 +1535,8 @@ static void ps_remove(struct hid_device *hdev) static const struct hid_device_id ps_devices[] = { { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS5_CONTROLLER) }, { HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS5_CONTROLLER) }, + { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS5_CONTROLLER_2) }, + { HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS5_CONTROLLER_2) }, { } }; MODULE_DEVICE_TABLE(hid, ps_devices); @@ -1330,6 +1547,9 @@ static struct hid_driver ps_driver = { .probe = ps_probe, .remove = ps_remove, .raw_event = ps_raw_event, + .driver = { + .dev_groups = ps_device_groups, + }, }; static int __init ps_init(void) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index e864288a6e0e..46f641481ac9 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -458,7 +458,7 @@ static void mtk_iommu_domain_free(struct iommu_domain *domain) static int mtk_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { - struct mtk_iommu_data *data = dev_iommu_priv_get(dev); + struct mtk_iommu_data *data = dev_iommu_priv_get(dev), *frstdata; struct mtk_iommu_domain *dom = to_mtk_domain(domain); struct device *m4udev = data->dev; int ret, domid; @@ -468,20 +468,24 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, return domid; if (!dom->data) { - if (mtk_iommu_domain_finalise(dom, data, domid)) + /* Data is in the frstdata in sharing pgtable case. */ + frstdata = mtk_iommu_get_m4u_data(); + + if (mtk_iommu_domain_finalise(dom, frstdata, domid)) return -ENODEV; dom->data = data; } + mutex_lock(&data->mutex); if (!data->m4u_dom) { /* Initialize the M4U HW */ ret = pm_runtime_resume_and_get(m4udev); if (ret < 0) - return ret; + goto err_unlock; ret = mtk_iommu_hw_init(data); if (ret) { pm_runtime_put(m4udev); - return ret; + goto err_unlock; } data->m4u_dom = dom; writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, @@ -489,9 +493,14 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, pm_runtime_put(m4udev); } + mutex_unlock(&data->mutex); mtk_iommu_config(data, dev, true, domid); return 0; + +err_unlock: + mutex_unlock(&data->mutex); + return ret; } static void mtk_iommu_detach_device(struct iommu_domain *domain, @@ -603,6 +612,7 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) if (domid < 0) return ERR_PTR(domid); + mutex_lock(&data->mutex); group = data->m4u_group[domid]; if (!group) { group = iommu_group_alloc(); @@ -611,6 +621,7 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) } else { iommu_group_ref_get(group); } + mutex_unlock(&data->mutex); return group; } @@ -884,6 +895,7 @@ static int mtk_iommu_probe(struct platform_device *pdev) } platform_set_drvdata(pdev, data); + mutex_init(&data->mutex); ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, "mtk-iommu.%pa", &ioaddr); @@ -934,7 +946,6 @@ static int mtk_iommu_remove(struct platform_device *pdev) list_del(&data->list); - clk_disable_unprepare(data->bclk); device_link_remove(data->smicomm_dev, &pdev->dev); pm_runtime_disable(&pdev->dev); devm_free_irq(&pdev->dev, data->irq, data); diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index f81fa8862ed0..f413546ac6e5 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -80,6 +80,8 @@ struct mtk_iommu_data { struct dma_iommu_mapping *mapping; /* For mtk_iommu_v1.c */ + struct mutex mutex; /* Protect m4u_group/m4u_dom above */ + struct list_head list; struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX]; }; diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 2a6de4858cac..8b3f0c9ddfc8 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3002,18 +3002,12 @@ static int __init allocate_lpi_tables(void) return 0; } -static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) +static u64 read_vpend_dirty_clear(void __iomem *vlpi_base) { u32 count = 1000000; /* 1s! */ bool clean; u64 val; - val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); - val &= ~GICR_VPENDBASER_Valid; - val &= ~clr; - val |= set; - gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); - do { val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); clean = !(val & GICR_VPENDBASER_Dirty); @@ -3024,10 +3018,26 @@ static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) } } while (!clean && count); - if (unlikely(val & GICR_VPENDBASER_Dirty)) { + if (unlikely(!clean)) pr_err_ratelimited("ITS virtual pending table not cleaning\n"); + + return val; +} + +static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) +{ + u64 val; + + /* Make sure we wait until the RD is done with the initial scan */ + val = read_vpend_dirty_clear(vlpi_base); + val &= ~GICR_VPENDBASER_Valid; + val &= ~clr; + val |= set; + gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); + + val = read_vpend_dirty_clear(vlpi_base); + if (unlikely(val & GICR_VPENDBASER_Dirty)) val |= GICR_VPENDBASER_PendingLast; - } return val; } diff --git a/drivers/md/dm-user.c b/drivers/md/dm-user.c index e5a85202d8a0..a8d771855e79 100644 --- a/drivers/md/dm-user.c +++ b/drivers/md/dm-user.c @@ -188,7 +188,6 @@ static void message_kill(struct message *m, mempool_t *pool) { m->bio->bi_status = BLK_STS_IOERR; bio_endio(m->bio); - bio_put(m->bio); mempool_free(m, pool); } @@ -989,7 +988,6 @@ finish_bio: */ WARN_ON(bio_size(c->cur_from_user->bio) != 0); bio_endio(c->cur_from_user->bio); - bio_put(c->cur_from_user->bio); /* * We don't actually need to take the target lock here, as all @@ -1227,7 +1225,6 @@ static int user_map(struct dm_target *ti, struct bio *bio) return DM_MAPIO_REQUEUE; } - bio_get(bio); entry->msg.type = bio_type_to_user_type(bio); entry->msg.flags = bio_flags_to_user_flags(bio); entry->msg.sector = bio->bi_iter.bi_sector; diff --git a/drivers/regulator/scmi-regulator.c b/drivers/regulator/scmi-regulator.c index 14d846bbf0bd..c48f7e134465 100644 --- a/drivers/regulator/scmi-regulator.c +++ b/drivers/regulator/scmi-regulator.c @@ -350,7 +350,7 @@ static int scmi_regulator_probe(struct scmi_device *sdev) if (ret == -ENOMEM) return ret; } - + of_node_put(np); /* * Register a regulator for each valid regulator-DT-entry that we * can successfully reach via SCMI and has a valid associated voltage diff --git a/drivers/scsi/ufs/ufs-mediatek.c b/drivers/scsi/ufs/ufs-mediatek.c index ca0ebb2c9b96..99eec4d06dde 100644 --- a/drivers/scsi/ufs/ufs-mediatek.c +++ b/drivers/scsi/ufs/ufs-mediatek.c @@ -501,7 +501,7 @@ static void ufs_mtk_init_va09_pwr_ctrl(struct ufs_hba *hba) struct ufs_mtk_host *host = ufshcd_get_variant(hba); host->reg_va09 = regulator_get(hba->dev, "va09"); - if (!host->reg_va09) + if (IS_ERR(host->reg_va09)) dev_info(hba->dev, "failed to get va09"); else host->caps |= UFS_MTK_CAP_VA09_PWR_CTRL; diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index a611d05b644a..ea90dec13695 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -2093,12 +2093,13 @@ void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag) lrbp->issue_time_stamp = ktime_get(); lrbp->compl_time_stamp = ktime_set(0, 0); - ufshcd_vops_setup_xfer_req(hba, task_tag, (lrbp->cmd ? true : false)); trace_android_vh_ufs_send_command(hba, lrbp); ufshcd_add_command_trace(hba, task_tag, "send"); ufshcd_clk_scaling_start_busy(hba); if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) ufshcd_start_monitor(hba, lrbp); + if (hba->vops && hba->vops->setup_xfer_req) + hba->vops->setup_xfer_req(hba, task_tag, !!lrbp->cmd); if (ufshcd_has_utrlcnr(hba)) { set_bit(task_tag, &hba->outstanding_reqs); ufshcd_writel(hba, 1 << task_tag, diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index 53678a8b80e5..6b80ac49908c 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -1304,18 +1304,6 @@ static inline int ufshcd_vops_pwr_change_notify(struct ufs_hba *hba, return -ENOTSUPP; } -static inline void ufshcd_vops_setup_xfer_req(struct ufs_hba *hba, int tag, - bool is_scsi_cmd) -{ - if (hba->vops && hba->vops->setup_xfer_req) { - unsigned long flags; - - spin_lock_irqsave(hba->host->host_lock, flags); - hba->vops->setup_xfer_req(hba, tag, is_scsi_cmd); - spin_unlock_irqrestore(hba->host->host_lock, flags); - } -} - static inline void ufshcd_vops_setup_task_mgmt(struct ufs_hba *hba, int tag, u8 tm_function) { diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 83a850038c51..40d358ca8b58 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -301,6 +301,7 @@ int dwc3_core_soft_reset(struct dwc3 *dwc) udelay(1); } while (--retries); + dev_warn(dwc->dev, "DWC3 controller soft reset failed.\n"); return -ETIMEDOUT; done: diff --git a/drivers/usb/dwc3/dwc3-qcom.c b/drivers/usb/dwc3/dwc3-qcom.c index ca3a35fd8f74..c711528f1995 100644 --- a/drivers/usb/dwc3/dwc3-qcom.c +++ b/drivers/usb/dwc3/dwc3-qcom.c @@ -350,7 +350,7 @@ static void dwc3_qcom_enable_interrupts(struct dwc3_qcom *qcom) } } -static int dwc3_qcom_suspend(struct dwc3_qcom *qcom) +static int dwc3_qcom_suspend(struct dwc3_qcom *qcom, bool wakeup) { u32 val; int i, ret; @@ -369,7 +369,7 @@ static int dwc3_qcom_suspend(struct dwc3_qcom *qcom) if (ret) dev_warn(qcom->dev, "failed to disable interconnect: %d\n", ret); - if (device_may_wakeup(qcom->dev)) + if (wakeup) dwc3_qcom_enable_interrupts(qcom); qcom->is_suspended = true; @@ -377,7 +377,7 @@ static int dwc3_qcom_suspend(struct dwc3_qcom *qcom) return 0; } -static int dwc3_qcom_resume(struct dwc3_qcom *qcom) +static int dwc3_qcom_resume(struct dwc3_qcom *qcom, bool wakeup) { int ret; int i; @@ -385,7 +385,7 @@ static int dwc3_qcom_resume(struct dwc3_qcom *qcom) if (!qcom->is_suspended) return 0; - if (device_may_wakeup(qcom->dev)) + if (wakeup) dwc3_qcom_disable_interrupts(qcom); for (i = 0; i < qcom->num_clocks; i++) { @@ -885,9 +885,11 @@ static int dwc3_qcom_remove(struct platform_device *pdev) static int __maybe_unused dwc3_qcom_pm_suspend(struct device *dev) { struct dwc3_qcom *qcom = dev_get_drvdata(dev); + bool wakeup = device_may_wakeup(dev); int ret = 0; - ret = dwc3_qcom_suspend(qcom); + + ret = dwc3_qcom_suspend(qcom, wakeup); if (!ret) qcom->pm_suspended = true; @@ -897,9 +899,10 @@ static int __maybe_unused dwc3_qcom_pm_suspend(struct device *dev) static int __maybe_unused dwc3_qcom_pm_resume(struct device *dev) { struct dwc3_qcom *qcom = dev_get_drvdata(dev); + bool wakeup = device_may_wakeup(dev); int ret; - ret = dwc3_qcom_resume(qcom); + ret = dwc3_qcom_resume(qcom, wakeup); if (!ret) qcom->pm_suspended = false; @@ -910,14 +913,14 @@ static int __maybe_unused dwc3_qcom_runtime_suspend(struct device *dev) { struct dwc3_qcom *qcom = dev_get_drvdata(dev); - return dwc3_qcom_suspend(qcom); + return dwc3_qcom_suspend(qcom, true); } static int __maybe_unused dwc3_qcom_runtime_resume(struct device *dev) { struct dwc3_qcom *qcom = dev_get_drvdata(dev); - return dwc3_qcom_resume(qcom); + return dwc3_qcom_resume(qcom, true); } static const struct dev_pm_ops dwc3_qcom_dev_pm_ops = { diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index 1064be5518f6..7f01513df0f2 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -218,7 +218,7 @@ out: return ret; } -static void dwc3_ep0_stall_and_restart(struct dwc3 *dwc) +void dwc3_ep0_stall_and_restart(struct dwc3 *dwc) { struct dwc3_ep *dep; @@ -239,6 +239,8 @@ static void dwc3_ep0_stall_and_restart(struct dwc3 *dwc) dwc3_gadget_giveback(dep, req, -ECONNRESET); } + dwc->eps[0]->trb_enqueue = 0; + dwc->eps[1]->trb_enqueue = 0; dwc->ep0state = EP0_SETUP_PHASE; dwc3_ep0_out_start(dwc); } @@ -813,7 +815,7 @@ static void dwc3_ep0_inspect_setup(struct dwc3 *dwc, int ret = -EINVAL; u32 len; - if (!dwc->gadget_driver) + if (!dwc->gadget_driver || !dwc->connected) goto out; trace_dwc3_ctrl_req(ctrl); @@ -1087,13 +1089,18 @@ void dwc3_ep0_send_delayed_status(struct dwc3 *dwc) __dwc3_ep0_do_control_status(dwc, dwc->eps[direction]); } -static void dwc3_ep0_end_control_data(struct dwc3 *dwc, struct dwc3_ep *dep) +void dwc3_ep0_end_control_data(struct dwc3 *dwc, struct dwc3_ep *dep) { struct dwc3_gadget_ep_cmd_params params; u32 cmd; int ret; - if (!dep->resource_index) + /* + * For status/DATA OUT stage, TRB will be queued on ep0 out + * endpoint for which resource index is zero. Hence allow + * queuing ENDXFER command for ep0 out endpoint. + */ + if (!dep->resource_index && dep->number) return; cmd = DWC3_DEPCMD_ENDTRANSFER; @@ -1134,6 +1141,11 @@ static void dwc3_ep0_xfernotready(struct dwc3 *dwc, if (dwc->ep0_next_event != DWC3_EP0_NRDY_STATUS) return; + if (dwc->setup_packet_pending) { + dwc3_ep0_stall_and_restart(dwc); + return; + } + dwc->ep0state = EP0_STATUS_PHASE; if (dwc->delayed_status) { diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 437f2a17a3b6..71625ddba11a 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -332,9 +332,17 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, } } - dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); - dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); - dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); + /* + * For some commands such as Update Transfer command, DEPCMDPARn + * registers are reserved. Since the driver often sends Update Transfer + * command, don't write to DEPCMDPARn to avoid register write delays and + * improve performance. + */ + if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) { + dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); + dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); + dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); + } /* * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're @@ -358,6 +366,12 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, cmd |= DWC3_DEPCMD_CMDACT; dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); + + if (!(cmd & DWC3_DEPCMD_CMDACT)) { + ret = 0; + goto skip_status; + } + do { reg = dwc3_readl(dep->regs, DWC3_DEPCMD); if (!(reg & DWC3_DEPCMD_CMDACT)) { @@ -399,6 +413,7 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, cmd_status = -ETIMEDOUT; } +skip_status: trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status); if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) { @@ -870,12 +885,13 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action) reg |= DWC3_DALEPENA_EP(dep->number); dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); + dep->trb_dequeue = 0; + dep->trb_enqueue = 0; + if (usb_endpoint_xfer_control(desc)) goto out; /* Initialize the TRB ring */ - dep->trb_dequeue = 0; - dep->trb_enqueue = 0; memset(dep->trb_pool, 0, sizeof(struct dwc3_trb) * DWC3_TRB_NUM); @@ -1638,6 +1654,40 @@ static int __dwc3_gadget_get_frame(struct dwc3 *dwc) return DWC3_DSTS_SOFFN(reg); } +/** + * __dwc3_stop_active_transfer - stop the current active transfer + * @dep: isoc endpoint + * @force: set forcerm bit in the command + * @interrupt: command complete interrupt after End Transfer command + * + * When setting force, the ForceRM bit will be set. In that case + * the controller won't update the TRB progress on command + * completion. It also won't clear the HWO bit in the TRB. + * The command will also not complete immediately in that case. + */ +static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt) +{ + struct dwc3_gadget_ep_cmd_params params; + u32 cmd; + int ret; + + cmd = DWC3_DEPCMD_ENDTRANSFER; + cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0; + cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0; + cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); + memset(¶ms, 0, sizeof(params)); + ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); + WARN_ON_ONCE(ret); + dep->resource_index = 0; + + if (!interrupt) + dep->flags &= ~DWC3_EP_TRANSFER_STARTED; + else if (!ret) + dep->flags |= DWC3_EP_END_TRANSFER_PENDING; + + return ret; +} + /** * dwc3_gadget_start_isoc_quirk - workaround invalid frame number * @dep: isoc endpoint @@ -1813,21 +1863,8 @@ static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep) * status, issue END_TRANSFER command and retry on the next XferNotReady * event. */ - if (ret == -EAGAIN) { - struct dwc3_gadget_ep_cmd_params params; - u32 cmd; - - cmd = DWC3_DEPCMD_ENDTRANSFER | - DWC3_DEPCMD_CMDIOC | - DWC3_DEPCMD_PARAM(dep->resource_index); - - dep->resource_index = 0; - memset(¶ms, 0, sizeof(params)); - - ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); - if (!ret) - dep->flags |= DWC3_EP_END_TRANSFER_PENDING; - } + if (ret == -EAGAIN) + ret = __dwc3_stop_active_transfer(dep, false, true); return ret; } @@ -1885,13 +1922,11 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) * errors which will force us issue EndTransfer command. */ if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { - if (!(dep->flags & DWC3_EP_PENDING_REQUEST) && - !(dep->flags & DWC3_EP_TRANSFER_STARTED)) - return 0; - - if ((dep->flags & DWC3_EP_PENDING_REQUEST)) { - if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) + if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) { + if ((dep->flags & DWC3_EP_PENDING_REQUEST)) return __dwc3_gadget_start_isoc(dep); + + return 0; } } @@ -2011,16 +2046,6 @@ static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, if (r == req) { struct dwc3_request *t; - /* - * If a Setup packet is received but yet to DMA out, the controller will - * not process the End Transfer command of any endpoint. Polling of its - * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a - * timeout. Delay issuing the End Transfer command until the Setup TRB is - * prepared. - */ - if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) - dep->flags |= DWC3_EP_DELAY_STOP; - /* wait until it is processed */ dwc3_stop_active_transfer(dep, true, true); @@ -2459,6 +2484,23 @@ static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc) spin_lock_irqsave(&dwc->lock, flags); dwc->connected = false; + /* + * Per databook, when we want to stop the gadget, if a control transfer + * is still in process, complete it and get the core into setup phase. + */ + if (dwc->ep0state != EP0_SETUP_PHASE) { + int ret; + + reinit_completion(&dwc->ep0_in_setup); + + spin_unlock_irqrestore(&dwc->lock, flags); + ret = wait_for_completion_timeout(&dwc->ep0_in_setup, + msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT)); + spin_lock_irqsave(&dwc->lock, flags); + if (ret == 0) + dev_warn(dwc->dev, "timed out waiting for SETUP phase\n"); + } + /* * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a * Section 4.1.8 Table 4-7, it states that for a device-initiated @@ -2490,19 +2532,6 @@ static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) vdwc->softconnect = is_on; - /* - * Per databook, when we want to stop the gadget, if a control transfer - * is still in process, complete it and get the core into setup phase. - */ - if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) { - reinit_completion(&dwc->ep0_in_setup); - - ret = wait_for_completion_timeout(&dwc->ep0_in_setup, - msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT)); - if (ret == 0) - dev_warn(dwc->dev, "timed out waiting for SETUP phase\n"); - } - /* * Avoid issuing a runtime resume if the device is already in the * suspended state during gadget disconnect. DWC3 gadget was already @@ -2688,6 +2717,7 @@ static int __dwc3_gadget_start(struct dwc3 *dwc) /* begin to receive SETUP packets */ dwc->ep0state = EP0_SETUP_PHASE; + dwc->ep0_bounced = false; dwc->link_state = DWC3_LINK_STATE_SS_DIS; dwc->delayed_status = false; dwc3_ep0_out_start(dwc); @@ -2936,6 +2966,7 @@ static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep) struct dwc3 *dwc = dep->dwc; u32 mdwidth; int size; + int maxpacket; mdwidth = dwc3_mdwidth(dwc); @@ -2948,21 +2979,24 @@ static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep) else size = DWC31_GTXFIFOSIZ_TXFDEP(size); - /* FIFO Depth is in MDWDITH bytes. Multiply */ - size *= mdwidth; - /* - * To meet performance requirement, a minimum TxFIFO size of 3x - * MaxPacketSize is recommended for endpoints that support burst and a - * minimum TxFIFO size of 2x MaxPacketSize for endpoints that don't - * support burst. Use those numbers and we can calculate the max packet - * limit as below. + * maxpacket size is determined as part of the following, after assuming + * a mult value of one maxpacket: + * DWC3 revision 280A and prior: + * fifo_size = mult * (max_packet / mdwidth) + 1; + * maxpacket = mdwidth * (fifo_size - 1); + * + * DWC3 revision 290A and onwards: + * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 + * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth; */ - if (dwc->maximum_speed >= USB_SPEED_SUPER) - size /= 3; + if (DWC3_VER_IS_PRIOR(DWC3, 290A)) + maxpacket = mdwidth * (size - 1); else - size /= 2; + maxpacket = mdwidth * ((size - 1) - 1) - mdwidth; + /* Functionally, space for one max packet is sufficient */ + size = min_t(int, maxpacket, 1024); usb_ep_set_maxpacket_limit(&dep->endpoint, size); dep->endpoint.max_streams = 16; @@ -3638,15 +3672,34 @@ static void dwc3_reset_gadget(struct dwc3 *dwc) void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt) { - struct dwc3_gadget_ep_cmd_params params; - u32 cmd; - int ret; + struct dwc3 *dwc = dep->dwc; + + /* + * Only issue End Transfer command to the control endpoint of a started + * Data Phase. Typically we should only do so in error cases such as + * invalid/unexpected direction as described in the control transfer + * flow of the programming guide. + */ + if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE) + return; if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || (dep->flags & DWC3_EP_DELAY_STOP) || (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) return; + /* + * If a Setup packet is received but yet to DMA out, the controller will + * not process the End Transfer command of any endpoint. Polling of its + * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a + * timeout. Delay issuing the End Transfer command until the Setup TRB is + * prepared. + */ + if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) { + dep->flags |= DWC3_EP_DELAY_STOP; + return; + } + /* * NOTICE: We are violating what the Databook says about the * EndTransfer command. Ideally we would _always_ wait for the @@ -3673,20 +3726,7 @@ void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, * * This mode is NOT available on the DWC_usb31 IP. */ - - cmd = DWC3_DEPCMD_ENDTRANSFER; - cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0; - cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0; - cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); - memset(¶ms, 0, sizeof(params)); - ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); - WARN_ON_ONCE(ret); - dep->resource_index = 0; - - if (!interrupt) - dep->flags &= ~DWC3_EP_TRANSFER_STARTED; - else - dep->flags |= DWC3_EP_END_TRANSFER_PENDING; + __dwc3_stop_active_transfer(dep, force, interrupt); } EXPORT_SYMBOL_GPL(dwc3_stop_active_transfer); @@ -3723,13 +3763,24 @@ static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc) reg &= ~DWC3_DCTL_INITU2ENA; dwc3_gadget_dctl_write_safe(dwc, reg); + dwc->connected = false; + dwc3_disconnect_gadget(dwc); dwc->gadget->speed = USB_SPEED_UNKNOWN; dwc->setup_packet_pending = false; usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); - dwc->connected = false; + if (dwc->ep0state != EP0_SETUP_PHASE) { + unsigned int dir; + + dir = !!dwc->ep0_expect_in; + if (dwc->ep0state == EP0_DATA_PHASE) + dwc3_ep0_end_control_data(dwc, dwc->eps[dir]); + else + dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]); + dwc3_ep0_stall_and_restart(dwc); + } } static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) @@ -3777,6 +3828,27 @@ static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) } dwc3_reset_gadget(dwc); + + /* + * From SNPS databook section 8.1.2, the EP0 should be in setup + * phase. So ensure that EP0 is in setup phase by issuing a stall + * and restart if EP0 is not in setup phase. + */ + if (dwc->ep0state != EP0_SETUP_PHASE) { + unsigned int dir; + + dir = !!dwc->ep0_expect_in; + if (dwc->ep0state == EP0_DATA_PHASE) + dwc3_ep0_end_control_data(dwc, dwc->eps[dir]); + else + dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]); + + dwc->eps[0]->trb_enqueue = 0; + dwc->eps[1]->trb_enqueue = 0; + + dwc3_ep0_stall_and_restart(dwc); + } + /* * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW @@ -4157,7 +4229,6 @@ static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt) struct dwc3 *dwc = evt->dwc; irqreturn_t ret = IRQ_NONE; int left; - u32 reg; left = evt->count; @@ -4188,9 +4259,8 @@ static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt) ret = IRQ_HANDLED; /* Unmask interrupt */ - reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0)); - reg &= ~DWC3_GEVNTSIZ_INTMASK; - dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg); + dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), + DWC3_GEVNTSIZ_SIZE(evt->length)); if (dwc->imod_interval) { dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); @@ -4224,7 +4294,6 @@ static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt) struct dwc3 *dwc = evt->dwc; u32 amount; u32 count; - u32 reg; if (pm_runtime_suspended(dwc->dev)) { pm_runtime_get(dwc->dev); @@ -4251,9 +4320,8 @@ static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt) evt->flags |= DWC3_EVENT_PENDING; /* Mask interrupt */ - reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0)); - reg |= DWC3_GEVNTSIZ_INTMASK; - dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg); + dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), + DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length)); amount = min(count, evt->length - evt->lpos); memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount); diff --git a/drivers/usb/dwc3/gadget.h b/drivers/usb/dwc3/gadget.h index f763380e672e..55a56cf67d73 100644 --- a/drivers/usb/dwc3/gadget.h +++ b/drivers/usb/dwc3/gadget.h @@ -110,6 +110,8 @@ void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req, void dwc3_ep0_interrupt(struct dwc3 *dwc, const struct dwc3_event_depevt *event); void dwc3_ep0_out_start(struct dwc3 *dwc); +void dwc3_ep0_end_control_data(struct dwc3 *dwc, struct dwc3_ep *dep); +void dwc3_ep0_stall_and_restart(struct dwc3 *dwc); int __dwc3_gadget_ep0_set_halt(struct usb_ep *ep, int value); int dwc3_gadget_ep0_set_halt(struct usb_ep *ep, int value); int dwc3_gadget_ep0_queue(struct usb_ep *ep, struct usb_request *request, diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c index 21dd268175c6..b6f86463a26e 100644 --- a/drivers/usb/gadget/configfs.c +++ b/drivers/usb/gadget/configfs.c @@ -16,7 +16,7 @@ #include #ifdef CONFIG_USB_CONFIGFS_F_ACC -extern int acc_ctrlrequest(struct usb_composite_dev *cdev, +extern int acc_ctrlrequest_composite(struct usb_composite_dev *cdev, const struct usb_ctrlrequest *ctrl); void acc_disconnect(void); #endif @@ -1572,7 +1572,7 @@ static int android_setup(struct usb_gadget *gadget, #ifdef CONFIG_USB_CONFIGFS_F_ACC if (value < 0) - value = acc_ctrlrequest(cdev, c); + value = acc_ctrlrequest_composite(cdev, c); #endif if (value < 0) diff --git a/drivers/usb/gadget/function/f_accessory.c b/drivers/usb/gadget/function/f_accessory.c index 3510f6d39f0c..7d35d6c58e47 100644 --- a/drivers/usb/gadget/function/f_accessory.c +++ b/drivers/usb/gadget/function/f_accessory.c @@ -1085,6 +1085,26 @@ err: } EXPORT_SYMBOL_GPL(acc_ctrlrequest); +int acc_ctrlrequest_composite(struct usb_composite_dev *cdev, + const struct usb_ctrlrequest *ctrl) +{ + u16 w_length = le16_to_cpu(ctrl->wLength); + + if (w_length > USB_COMP_EP0_BUFSIZ) { + if (ctrl->bRequestType & USB_DIR_IN) { + /* Cast away the const, we are going to overwrite on purpose. */ + __le16 *temp = (__le16 *)&ctrl->wLength; + + *temp = cpu_to_le16(USB_COMP_EP0_BUFSIZ); + w_length = USB_COMP_EP0_BUFSIZ; + } else { + return -EINVAL; + } + } + return acc_ctrlrequest(cdev, ctrl); +} +EXPORT_SYMBOL_GPL(acc_ctrlrequest_composite); + static int __acc_function_bind(struct usb_configuration *c, struct usb_function *f, bool configfs) diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index ecf298bc49ed..f49f8706fbd3 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -279,6 +279,9 @@ static int __ffs_ep0_queue_wait(struct ffs_data *ffs, char *data, size_t len) struct usb_request *req = ffs->ep0req; int ret; + if (!req) + return -EINVAL; + req->zero = len < le16_to_cpu(ffs->ev.setup.wLength); spin_unlock_irq(&ffs->ev.waitq.lock); @@ -1891,10 +1894,14 @@ static void functionfs_unbind(struct ffs_data *ffs) ENTER(); if (!WARN_ON(!ffs->gadget)) { + /* dequeue before freeing ep0req */ + usb_ep_dequeue(ffs->gadget->ep0, ffs->ep0req); + mutex_lock(&ffs->mutex); usb_ep_free_request(ffs->gadget->ep0, ffs->ep0req); ffs->ep0req = NULL; ffs->gadget = NULL; clear_bit(FFS_FL_BOUND, &ffs->flags); + mutex_unlock(&ffs->mutex); ffs_data_put(ffs); } } diff --git a/drivers/usb/gadget/function/f_mass_storage.c b/drivers/usb/gadget/function/f_mass_storage.c index 12fe0be35528..7790fbed443c 100644 --- a/drivers/usb/gadget/function/f_mass_storage.c +++ b/drivers/usb/gadget/function/f_mass_storage.c @@ -1191,13 +1191,14 @@ static int do_read_toc(struct fsg_common *common, struct fsg_buffhd *bh) u8 format; int i, len; + format = common->cmnd[2] & 0xf; + if ((common->cmnd[1] & ~0x02) != 0 || /* Mask away MSF */ - start_track > 1) { + (start_track > 1 && format != 0x1)) { curlun->sense_data = SS_INVALID_FIELD_IN_CDB; return -EINVAL; } - format = common->cmnd[2] & 0xf; /* * Check if CDB is old style SFF-8020i * i.e. format is in 2 MSBs of byte 9 @@ -1207,8 +1208,8 @@ static int do_read_toc(struct fsg_common *common, struct fsg_buffhd *bh) format = (common->cmnd[9] >> 6) & 0x3; switch (format) { - case 0: - /* Formatted TOC */ + case 0: /* Formatted TOC */ + case 1: /* Multi-session info */ len = 4 + 2*8; /* 4 byte header + 2 descriptors */ memset(buf, 0, len); buf[1] = len - 2; /* TOC Length excludes length field */ @@ -1249,7 +1250,7 @@ static int do_read_toc(struct fsg_common *common, struct fsg_buffhd *bh) return len; default: - /* Multi-session, PMA, ATIP, CD-TEXT not supported/required */ + /* PMA, ATIP, CD-TEXT not supported/required */ curlun->sense_data = SS_INVALID_FIELD_IN_CDB; return -EINVAL; } diff --git a/drivers/usb/gadget/function/f_uac2.c b/drivers/usb/gadget/function/f_uac2.c index da5250a415d5..91affe8babb8 100644 --- a/drivers/usb/gadget/function/f_uac2.c +++ b/drivers/usb/gadget/function/f_uac2.c @@ -281,6 +281,12 @@ static struct usb_endpoint_descriptor ss_ep_int_desc = { .bInterval = 4, }; +static struct usb_ss_ep_comp_descriptor ss_ep_int_desc_comp = { + .bLength = sizeof(ss_ep_int_desc_comp), + .bDescriptorType = USB_DT_SS_ENDPOINT_COMP, + .wBytesPerInterval = cpu_to_le16(6), +}; + /* Audio Streaming OUT Interface - Alt0 */ static struct usb_interface_descriptor std_as_out_if0_desc = { .bLength = sizeof std_as_out_if0_desc, @@ -594,7 +600,8 @@ static struct usb_descriptor_header *ss_audio_desc[] = { (struct usb_descriptor_header *)&in_feature_unit_desc, (struct usb_descriptor_header *)&io_out_ot_desc, - (struct usb_descriptor_header *)&ss_ep_int_desc, + (struct usb_descriptor_header *)&ss_ep_int_desc, + (struct usb_descriptor_header *)&ss_ep_int_desc_comp, (struct usb_descriptor_header *)&std_as_out_if0_desc, (struct usb_descriptor_header *)&std_as_out_if1_desc, @@ -746,6 +753,7 @@ static void setup_headers(struct f_uac2_opts *opts, struct usb_ss_ep_comp_descriptor *epout_desc_comp = NULL; struct usb_ss_ep_comp_descriptor *epin_desc_comp = NULL; struct usb_ss_ep_comp_descriptor *epin_fback_desc_comp = NULL; + struct usb_ss_ep_comp_descriptor *ep_int_desc_comp = NULL; struct usb_endpoint_descriptor *epout_desc; struct usb_endpoint_descriptor *epin_desc; struct usb_endpoint_descriptor *epin_fback_desc; @@ -773,6 +781,7 @@ static void setup_headers(struct f_uac2_opts *opts, epin_fback_desc = &ss_epin_fback_desc; epin_fback_desc_comp = &ss_epin_fback_desc_comp; ep_int_desc = &ss_ep_int_desc; + ep_int_desc_comp = &ss_ep_int_desc_comp; } i = 0; @@ -801,8 +810,11 @@ static void setup_headers(struct f_uac2_opts *opts, if (EPOUT_EN(opts)) headers[i++] = USBDHDR(&io_out_ot_desc); - if (FUOUT_EN(opts) || FUIN_EN(opts)) - headers[i++] = USBDHDR(ep_int_desc); + if (FUOUT_EN(opts) || FUIN_EN(opts)) { + headers[i++] = USBDHDR(ep_int_desc); + if (ep_int_desc_comp) + headers[i++] = USBDHDR(ep_int_desc_comp); + } if (EPOUT_EN(opts)) { headers[i++] = USBDHDR(&std_as_out_if0_desc); diff --git a/drivers/usb/gadget/function/uvc_configfs.c b/drivers/usb/gadget/function/uvc_configfs.c index 9fafa8001d33..3f1e03ef40c9 100644 --- a/drivers/usb/gadget/function/uvc_configfs.c +++ b/drivers/usb/gadget/function/uvc_configfs.c @@ -2448,6 +2448,7 @@ static ssize_t f_uvc_opts_string_##cname##_store(struct config_item *item,\ const char *page, size_t len) \ { \ struct f_uvc_opts *opts = to_f_uvc_opts(item); \ + int size = min(sizeof(opts->aname), len + 1); \ int ret = 0; \ \ mutex_lock(&opts->lock); \ @@ -2456,8 +2457,9 @@ static ssize_t f_uvc_opts_string_##cname##_store(struct config_item *item,\ goto end; \ } \ \ - ret = snprintf(opts->aname, min(sizeof(opts->aname), len), \ - "%s", page); \ + ret = strscpy(opts->aname, page, size); \ + if (ret == -E2BIG) \ + ret = size - 1; \ \ end: \ mutex_unlock(&opts->lock); \ diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c index fb9c14d0d584..17cea37c73fd 100644 --- a/drivers/usb/typec/tcpm/tcpci.c +++ b/drivers/usb/typec/tcpm/tcpci.c @@ -85,9 +85,25 @@ static int tcpci_write16(struct tcpci *tcpci, unsigned int reg, u16 val) static int tcpci_set_cc(struct tcpc_dev *tcpc, enum typec_cc_status cc) { struct tcpci *tcpci = tcpc_to_tcpci(tcpc); + bool vconn_pres; + enum typec_cc_polarity polarity = TYPEC_POLARITY_CC1; unsigned int reg; int ret; + ret = regmap_read(tcpci->regmap, TCPC_POWER_STATUS, ®); + if (ret < 0) + return ret; + + vconn_pres = !!(reg & TCPC_POWER_STATUS_VCONN_PRES); + if (vconn_pres) { + ret = regmap_read(tcpci->regmap, TCPC_TCPC_CTRL, ®); + if (ret < 0) + return ret; + + if (reg & TCPC_TCPC_CTRL_ORIENTATION) + polarity = TYPEC_POLARITY_CC2; + } + switch (cc) { case TYPEC_CC_RA: reg = (TCPC_ROLE_CTRL_CC_RA << TCPC_ROLE_CTRL_CC1_SHIFT) | @@ -122,6 +138,16 @@ static int tcpci_set_cc(struct tcpc_dev *tcpc, enum typec_cc_status cc) break; } + if (vconn_pres) { + if (polarity == TYPEC_POLARITY_CC2) { + reg &= ~(TCPC_ROLE_CTRL_CC1_MASK << TCPC_ROLE_CTRL_CC1_SHIFT); + reg |= (TCPC_ROLE_CTRL_CC_OPEN << TCPC_ROLE_CTRL_CC1_SHIFT); + } else { + reg &= ~(TCPC_ROLE_CTRL_CC2_MASK << TCPC_ROLE_CTRL_CC2_SHIFT); + reg |= (TCPC_ROLE_CTRL_CC_OPEN << TCPC_ROLE_CTRL_CC2_SHIFT); + } + } + ret = regmap_write(tcpci->regmap, TCPC_ROLE_CTRL, reg); if (ret < 0) return ret; diff --git a/drivers/usb/typec/tcpm/tcpci.h b/drivers/usb/typec/tcpm/tcpci.h index 2be7a77d400e..b2edd45f13c6 100644 --- a/drivers/usb/typec/tcpm/tcpci.h +++ b/drivers/usb/typec/tcpm/tcpci.h @@ -98,6 +98,7 @@ #define TCPC_POWER_STATUS_SOURCING_VBUS BIT(4) #define TCPC_POWER_STATUS_VBUS_DET BIT(3) #define TCPC_POWER_STATUS_VBUS_PRES BIT(2) +#define TCPC_POWER_STATUS_VCONN_PRES BIT(1) #define TCPC_POWER_STATUS_SINKING_VBUS BIT(0) #define TCPC_FAULT_STATUS 0x1f diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 9724311c5fa2..dc0d3a48894a 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -5354,6 +5354,10 @@ static void _tcpm_pd_vbus_vsafe0v(struct tcpm_port *port) case PR_SWAP_SNK_SRC_SOURCE_ON: /* Do nothing, vsafe0v is expected during transition */ break; + case SNK_ATTACH_WAIT: + case SNK_DEBOUNCED: + /*Do nothing, still waiting for VSAFE5V for connect */ + break; default: if (port->pwr_role == TYPEC_SINK && port->auto_vbus_discharge_enabled) tcpm_set_state(port, SNK_UNATTACHED, 0); diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index f253310a92b4..516d00affa82 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -1061,6 +1061,15 @@ static int ucsi_register_port(struct ucsi *ucsi, int index) con->num = index + 1; con->ucsi = ucsi; + cap->fwnode = ucsi_find_fwnode(con); + con->usb_role_sw = fwnode_usb_role_switch_get(cap->fwnode); + if (IS_ERR(con->usb_role_sw)) { + dev_err(ucsi->dev, "con%d: failed to get usb role switch\n", + con->num); + return PTR_ERR(con->usb_role_sw); + } + + /* Delay other interactions with the con until registration is complete */ mutex_lock(&con->lock); @@ -1096,7 +1105,6 @@ static int ucsi_register_port(struct ucsi *ucsi, int index) if (con->cap.op_mode & UCSI_CONCAP_OPMODE_DEBUG_ACCESSORY) *accessory = TYPEC_ACCESSORY_DEBUG; - cap->fwnode = ucsi_find_fwnode(con); cap->driver_data = con; cap->ops = &ucsi_ops; @@ -1154,13 +1162,6 @@ static int ucsi_register_port(struct ucsi *ucsi, int index) ucsi_port_psy_changed(con); } - con->usb_role_sw = fwnode_usb_role_switch_get(cap->fwnode); - if (IS_ERR(con->usb_role_sw)) { - dev_err(ucsi->dev, "con%d: failed to get usb role switch\n", - con->num); - con->usb_role_sw = NULL; - } - /* Only notify USB controller if partner supports USB data */ if (!(UCSI_CONSTAT_PARTNER_FLAGS(con->status.flags) & UCSI_CONSTAT_PARTNER_FLAG_USB)) u_role = USB_ROLE_NONE; @@ -1273,12 +1274,21 @@ err: static void ucsi_init_work(struct work_struct *work) { - struct ucsi *ucsi = container_of(work, struct ucsi, work); + struct ucsi_android *aucsi = container_of(work, + struct ucsi_android, work.work); int ret; - ret = ucsi_init(ucsi); + ret = ucsi_init(&aucsi->ucsi); if (ret) - dev_err(ucsi->dev, "PPM init failed (%d)\n", ret); + dev_err(aucsi->ucsi.dev, "PPM init failed (%d)\n", ret); + + if (ret == -EPROBE_DEFER) { + if (aucsi->work_count++ > UCSI_ROLE_SWITCH_WAIT_COUNT) + return; + + queue_delayed_work(system_long_wq, &aucsi->work, + UCSI_ROLE_SWITCH_INTERVAL); + } } /** @@ -1310,15 +1320,17 @@ EXPORT_SYMBOL_GPL(ucsi_set_drvdata); struct ucsi *ucsi_create(struct device *dev, const struct ucsi_operations *ops) { struct ucsi *ucsi; + struct ucsi_android *aucsi; if (!ops || !ops->read || !ops->sync_write || !ops->async_write) return ERR_PTR(-EINVAL); - ucsi = kzalloc(sizeof(*ucsi), GFP_KERNEL); - if (!ucsi) + aucsi = kzalloc(sizeof(*aucsi), GFP_KERNEL); + if (!aucsi) return ERR_PTR(-ENOMEM); - INIT_WORK(&ucsi->work, ucsi_init_work); + ucsi = &aucsi->ucsi; + INIT_DELAYED_WORK(&aucsi->work, ucsi_init_work); mutex_init(&ucsi->ppm_lock); ucsi->dev = dev; ucsi->ops = ops; @@ -1333,7 +1345,9 @@ EXPORT_SYMBOL_GPL(ucsi_create); */ void ucsi_destroy(struct ucsi *ucsi) { - kfree(ucsi); + struct ucsi_android *aucsi = container_of(ucsi, + struct ucsi_android, ucsi); + kfree(aucsi); } EXPORT_SYMBOL_GPL(ucsi_destroy); @@ -1343,6 +1357,8 @@ EXPORT_SYMBOL_GPL(ucsi_destroy); */ int ucsi_register(struct ucsi *ucsi) { + struct ucsi_android *aucsi = container_of(ucsi, + struct ucsi_android, ucsi); int ret; ret = ucsi->ops->read(ucsi, UCSI_VERSION, &ucsi->version, @@ -1353,7 +1369,7 @@ int ucsi_register(struct ucsi *ucsi) if (!ucsi->version) return -ENODEV; - queue_work(system_long_wq, &ucsi->work); + queue_delayed_work(system_long_wq, &aucsi->work, 0); return 0; } @@ -1367,11 +1383,13 @@ EXPORT_SYMBOL_GPL(ucsi_register); */ void ucsi_unregister(struct ucsi *ucsi) { + struct ucsi_android *aucsi = container_of(ucsi, + struct ucsi_android, ucsi); u64 cmd = UCSI_SET_NOTIFICATION_ENABLE; int i; /* Make sure that we are not in the middle of driver initialization */ - cancel_work_sync(&ucsi->work); + cancel_delayed_work_sync(&aucsi->work); /* Disable notifications */ ucsi->ops->async_write(ucsi, UCSI_CONTROL, &cmd, sizeof(cmd)); diff --git a/drivers/usb/typec/ucsi/ucsi.h b/drivers/usb/typec/ucsi/ucsi.h index b3450d87f324..d366ab2add37 100644 --- a/drivers/usb/typec/ucsi/ucsi.h +++ b/drivers/usb/typec/ucsi/ucsi.h @@ -289,6 +289,9 @@ struct ucsi { struct ucsi_connector *connector; struct work_struct work; +#define UCSI_ROLE_SWITCH_RETRY_PER_HZ 10 +#define UCSI_ROLE_SWITCH_INTERVAL (HZ / UCSI_ROLE_SWITCH_RETRY_PER_HZ) +#define UCSI_ROLE_SWITCH_WAIT_COUNT (10 * UCSI_ROLE_SWITCH_RETRY_PER_HZ) /* PPM Communication lock */ struct mutex ppm_lock; @@ -304,6 +307,25 @@ struct ucsi { #define EVENT_PROCESSING 3 }; +/** + * struct ucsi_android - contains parameters without modifying the format + * of ucsi struct. + * @ucsi: contains the ucsi reference. + * @work: work structure for queuing ucsi_init_work. + * @work_count: to track the wait count(MAX= UCSI_ROLE_SWITCH_WAIT_COUNT). + * + * Required to address Bug: 260537721 + * If the role switch module probes late the + * fwnode_usb_role_switch_get() will fail with -EPROBE_DEFER. + * To recover from this, restart the ucsi_init_work + * to find the fwnode again using a delayed workqueue. + */ +struct ucsi_android { + struct ucsi ucsi; + struct delayed_work work; + int work_count; +}; + #define UCSI_MAX_SVID 5 #define UCSI_MAX_ALTMODES (UCSI_MAX_SVID * 6) diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 3026199266ef..6d221fafc5b9 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -3655,6 +3655,9 @@ static struct buffer_head *ext4_get_first_dir_block(handle_t *handle, struct buffer_head *bh; if (!ext4_has_inline_data(inode)) { + struct ext4_dir_entry_2 *de; + unsigned int offset; + /* The first directory block must not be a hole, so * treat it as DIRENT_HTREE */ @@ -3663,9 +3666,30 @@ static struct buffer_head *ext4_get_first_dir_block(handle_t *handle, *retval = PTR_ERR(bh); return NULL; } - *parent_de = ext4_next_entry( - (struct ext4_dir_entry_2 *)bh->b_data, - inode->i_sb->s_blocksize); + + de = (struct ext4_dir_entry_2 *) bh->b_data; + if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, + bh->b_size, 0, 0) || + le32_to_cpu(de->inode) != inode->i_ino || + strcmp(".", de->name)) { + EXT4_ERROR_INODE(inode, "directory missing '.'"); + brelse(bh); + *retval = -EFSCORRUPTED; + return NULL; + } + offset = ext4_rec_len_from_disk(de->rec_len, + inode->i_sb->s_blocksize); + de = ext4_next_entry(de, inode->i_sb->s_blocksize); + if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, + bh->b_size, 0, offset) || + le32_to_cpu(de->inode) == 0 || strcmp("..", de->name)) { + EXT4_ERROR_INODE(inode, "directory missing '..'"); + brelse(bh); + *retval = -EFSCORRUPTED; + return NULL; + } + *parent_de = de; + return bh; } diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 7a4abd303fa0..3116b38c2f6b 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -1469,6 +1469,7 @@ int f2fs_write_multi_pages(struct compress_ctx *cc, if (cluster_may_compress(cc)) { err = f2fs_compress_pages(cc); if (err == -EAGAIN) { + add_compr_block_stat(cc->inode, cc->cluster_size); goto write; } else if (err) { f2fs_put_rpages_wbc(cc, wbc, true, 1); @@ -1659,6 +1660,30 @@ void f2fs_put_page_dic(struct page *page) f2fs_put_dic(dic); } +/* + * check whether cluster blocks are contiguous, and add extent cache entry + * only if cluster blocks are logically and physically contiguous. + */ +unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn) +{ + bool compressed = f2fs_data_blkaddr(dn) == COMPRESS_ADDR; + int i = compressed ? 1 : 0; + block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_page, + dn->ofs_in_node + i); + + for (i += 1; i < F2FS_I(dn->inode)->i_cluster_size; i++) { + block_t blkaddr = data_blkaddr(dn->inode, dn->node_page, + dn->ofs_in_node + i); + + if (!__is_valid_data_blkaddr(blkaddr)) + break; + if (first_blkaddr + i - (compressed ? 1 : 0) != blkaddr) + return 0; + } + + return compressed ? i - 1 : i; +} + const struct address_space_operations f2fs_compress_aops = { .releasepage = f2fs_release_page, .invalidatepage = f2fs_invalidate_page, diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 419d9b4f8ff7..bc56bdd3e38b 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -1085,7 +1085,7 @@ void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) { dn->data_blkaddr = blkaddr; f2fs_set_data_blkaddr(dn); - f2fs_update_extent_cache(dn); + f2fs_update_read_extent_cache(dn); } /* dn->ofs_in_node will be returned with up-to-date last block pointer */ @@ -1151,10 +1151,10 @@ int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index) { - struct extent_info ei = {0, 0, 0}; + struct extent_info ei = {0, }; struct inode *inode = dn->inode; - if (f2fs_lookup_extent_cache(inode, index, &ei)) { + if (f2fs_lookup_read_extent_cache(inode, index, &ei)) { dn->data_blkaddr = ei.blk + index - ei.fofs; return 0; } @@ -1168,14 +1168,14 @@ struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index, struct address_space *mapping = inode->i_mapping; struct dnode_of_data dn; struct page *page; - struct extent_info ei = {0,0,0}; + struct extent_info ei = {0, }; int err; page = f2fs_grab_cache_page(mapping, index, for_write); if (!page) return ERR_PTR(-ENOMEM); - if (f2fs_lookup_extent_cache(inode, index, &ei)) { + if (f2fs_lookup_read_extent_cache(inode, index, &ei)) { dn.data_blkaddr = ei.blk + index - ei.fofs; if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr, DATA_GENERIC_ENHANCE_READ)) { @@ -1466,7 +1466,7 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int err = 0, ofs = 1; unsigned int ofs_in_node, last_ofs_in_node; blkcnt_t prealloc; - struct extent_info ei = {0,0,0}; + struct extent_info ei = {0, }; block_t blkaddr; unsigned int start_pgofs; @@ -1480,7 +1480,7 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, pgofs = (pgoff_t)map->m_lblk; end = pgofs + maxblocks; - if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) { + if (!create && f2fs_lookup_read_extent_cache(inode, pgofs, &ei)) { if (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO && map->m_may_create) goto next_dnode; @@ -1654,7 +1654,7 @@ skip: if (map->m_flags & F2FS_MAP_MAPPED) { unsigned int ofs = start_pgofs - map->m_lblk; - f2fs_update_extent_cache_range(&dn, + f2fs_update_read_extent_cache_range(&dn, start_pgofs, map->m_pblk + ofs, map->m_len - ofs); } @@ -1679,7 +1679,7 @@ sync_out: if (map->m_flags & F2FS_MAP_MAPPED) { unsigned int ofs = start_pgofs - map->m_lblk; - f2fs_update_extent_cache_range(&dn, + f2fs_update_read_extent_cache_range(&dn, start_pgofs, map->m_pblk + ofs, map->m_len - ofs); } @@ -2156,6 +2156,8 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, sector_t last_block_in_file; const unsigned blocksize = blks_to_bytes(inode, 1); struct decompress_io_ctx *dic = NULL; + struct extent_info ei = {}; + bool from_dnode = true; int i; int ret = 0; @@ -2188,6 +2190,12 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, if (f2fs_cluster_is_empty(cc)) goto out; + if (f2fs_lookup_read_extent_cache(inode, start_idx, &ei)) + from_dnode = false; + + if (!from_dnode) + goto skip_reading_dnode; + set_new_dnode(&dn, inode, NULL, NULL, 0); ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); if (ret) @@ -2195,11 +2203,13 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR); +skip_reading_dnode: for (i = 1; i < cc->cluster_size; i++) { block_t blkaddr; - blkaddr = data_blkaddr(dn.inode, dn.node_page, - dn.ofs_in_node + i); + blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_page, + dn.ofs_in_node + i) : + ei.blk + i - 1; if (!__is_valid_data_blkaddr(blkaddr)) break; @@ -2209,6 +2219,9 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, goto out_put_dnode; } cc->nr_cpages++; + + if (!from_dnode && i >= ei.c_len) + break; } /* nothing to decompress */ @@ -2228,8 +2241,9 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, block_t blkaddr; struct bio_post_read_ctx *ctx; - blkaddr = data_blkaddr(dn.inode, dn.node_page, - dn.ofs_in_node + i + 1); + blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_page, + dn.ofs_in_node + i + 1) : + ei.blk + i; f2fs_wait_on_block_writeback(inode, blkaddr); @@ -2274,13 +2288,15 @@ submit_and_realloc: *last_block_in_bio = blkaddr; } - f2fs_put_dnode(&dn); + if (from_dnode) + f2fs_put_dnode(&dn); *bio_ret = bio; return 0; out_put_dnode: - f2fs_put_dnode(&dn); + if (from_dnode) + f2fs_put_dnode(&dn); out: for (i = 0; i < cc->cluster_size; i++) { if (cc->rpages[i]) { @@ -2584,14 +2600,14 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) struct page *page = fio->page; struct inode *inode = page->mapping->host; struct dnode_of_data dn; - struct extent_info ei = {0,0,0}; + struct extent_info ei = {0, }; struct node_info ni; bool ipu_force = false; int err = 0; set_new_dnode(&dn, inode, NULL, NULL, 0); if (need_inplace_update(fio) && - f2fs_lookup_extent_cache(inode, page->index, &ei)) { + f2fs_lookup_read_extent_cache(inode, page->index, &ei)) { fio->old_blkaddr = ei.blk + page->index - ei.fofs; if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, @@ -3265,7 +3281,7 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi, struct dnode_of_data dn; struct page *ipage; bool locked = false; - struct extent_info ei = {0,0,0}; + struct extent_info ei = {0, }; int err = 0; int flag; @@ -3316,7 +3332,7 @@ restart: } else if (locked) { err = f2fs_get_block(&dn, index); } else { - if (f2fs_lookup_extent_cache(inode, index, &ei)) { + if (f2fs_lookup_read_extent_cache(inode, index, &ei)) { dn.data_blkaddr = ei.blk + index - ei.fofs; } else { /* hole case */ diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c index 53ed1e9191f0..f33e64acaf2b 100644 --- a/fs/f2fs/debug.c +++ b/fs/f2fs/debug.c @@ -72,15 +72,26 @@ static void update_general_status(struct f2fs_sb_info *sbi) si->main_area_zones = si->main_area_sections / le32_to_cpu(raw_super->secs_per_zone); - /* validation check of the segment numbers */ + /* general extent cache stats */ + for (i = 0; i < NR_EXTENT_CACHES; i++) { + struct extent_tree_info *eti = &sbi->extent_tree[i]; + + si->hit_cached[i] = atomic64_read(&sbi->read_hit_cached[i]); + si->hit_rbtree[i] = atomic64_read(&sbi->read_hit_rbtree[i]); + si->total_ext[i] = atomic64_read(&sbi->total_hit_ext[i]); + si->hit_total[i] = si->hit_cached[i] + si->hit_rbtree[i]; + si->ext_tree[i] = atomic_read(&eti->total_ext_tree); + si->zombie_tree[i] = atomic_read(&eti->total_zombie_tree); + si->ext_node[i] = atomic_read(&eti->total_ext_node); + } + /* read extent_cache only */ si->hit_largest = atomic64_read(&sbi->read_hit_largest); - si->hit_cached = atomic64_read(&sbi->read_hit_cached); - si->hit_rbtree = atomic64_read(&sbi->read_hit_rbtree); - si->hit_total = si->hit_largest + si->hit_cached + si->hit_rbtree; - si->total_ext = atomic64_read(&sbi->total_hit_ext); - si->ext_tree = atomic_read(&sbi->total_ext_tree); - si->zombie_tree = atomic_read(&sbi->total_zombie_tree); - si->ext_node = atomic_read(&sbi->total_ext_node); + si->hit_total[EX_READ] += si->hit_largest; + + /* block age extent_cache only */ + si->allocated_data_blocks = atomic64_read(&sbi->allocated_data_blocks); + + /* validation check of the segment numbers */ si->ndirty_node = get_pages(sbi, F2FS_DIRTY_NODES); si->ndirty_dent = get_pages(sbi, F2FS_DIRTY_DENTS); si->ndirty_meta = get_pages(sbi, F2FS_DIRTY_META); @@ -299,10 +310,16 @@ get_cache: si->cache_mem += si->inmem_pages * sizeof(struct inmem_pages); for (i = 0; i < MAX_INO_ENTRY; i++) si->cache_mem += sbi->im[i].ino_num * sizeof(struct ino_entry); - si->cache_mem += atomic_read(&sbi->total_ext_tree) * + + for (i = 0; i < NR_EXTENT_CACHES; i++) { + struct extent_tree_info *eti = &sbi->extent_tree[i]; + + si->ext_mem[i] = atomic_read(&eti->total_ext_tree) * sizeof(struct extent_tree); - si->cache_mem += atomic_read(&sbi->total_ext_node) * + si->ext_mem[i] += atomic_read(&eti->total_ext_node) * sizeof(struct extent_node); + si->cache_mem += si->ext_mem[i]; + } si->page_mem = 0; if (sbi->node_inode) { @@ -471,16 +488,34 @@ static int stat_show(struct seq_file *s, void *v) si->skipped_atomic_files[BG_GC]); seq_printf(s, "BG skip : IO: %u, Other: %u\n", si->io_skip_bggc, si->other_skip_bggc); - seq_puts(s, "\nExtent Cache:\n"); + seq_puts(s, "\nExtent Cache (Read):\n"); seq_printf(s, " - Hit Count: L1-1:%llu L1-2:%llu L2:%llu\n", - si->hit_largest, si->hit_cached, - si->hit_rbtree); + si->hit_largest, si->hit_cached[EX_READ], + si->hit_rbtree[EX_READ]); seq_printf(s, " - Hit Ratio: %llu%% (%llu / %llu)\n", - !si->total_ext ? 0 : - div64_u64(si->hit_total * 100, si->total_ext), - si->hit_total, si->total_ext); + !si->total_ext[EX_READ] ? 0 : + div64_u64(si->hit_total[EX_READ] * 100, + si->total_ext[EX_READ]), + si->hit_total[EX_READ], si->total_ext[EX_READ]); seq_printf(s, " - Inner Struct Count: tree: %d(%d), node: %d\n", - si->ext_tree, si->zombie_tree, si->ext_node); + si->ext_tree[EX_READ], si->zombie_tree[EX_READ], + si->ext_node[EX_READ]); + seq_puts(s, "\nExtent Cache (Block Age):\n"); + seq_printf(s, " - Allocated Data Blocks: %llu\n", + si->allocated_data_blocks); + seq_printf(s, " - Hit Count: L1:%llu L2:%llu\n", + si->hit_cached[EX_BLOCK_AGE], + si->hit_rbtree[EX_BLOCK_AGE]); + seq_printf(s, " - Hit Ratio: %llu%% (%llu / %llu)\n", + !si->total_ext[EX_BLOCK_AGE] ? 0 : + div64_u64(si->hit_total[EX_BLOCK_AGE] * 100, + si->total_ext[EX_BLOCK_AGE]), + si->hit_total[EX_BLOCK_AGE], + si->total_ext[EX_BLOCK_AGE]); + seq_printf(s, " - Inner Struct Count: tree: %d(%d), node: %d\n", + si->ext_tree[EX_BLOCK_AGE], + si->zombie_tree[EX_BLOCK_AGE], + si->ext_node[EX_BLOCK_AGE]); seq_puts(s, "\nBalancing F2FS Async:\n"); seq_printf(s, " - DIO (R: %4d, W: %4d)\n", si->nr_dio_read, si->nr_dio_write); @@ -546,8 +581,12 @@ static int stat_show(struct seq_file *s, void *v) (si->base_mem + si->cache_mem + si->page_mem) >> 10); seq_printf(s, " - static: %llu KB\n", si->base_mem >> 10); - seq_printf(s, " - cached: %llu KB\n", + seq_printf(s, " - cached all: %llu KB\n", si->cache_mem >> 10); + seq_printf(s, " - read extent cache: %llu KB\n", + si->ext_mem[EX_READ] >> 10); + seq_printf(s, " - block age extent cache: %llu KB\n", + si->ext_mem[EX_BLOCK_AGE] >> 10); seq_printf(s, " - paged : %llu KB\n", si->page_mem >> 10); } @@ -579,10 +618,15 @@ int f2fs_build_stats(struct f2fs_sb_info *sbi) si->sbi = sbi; sbi->stat_info = si; - atomic64_set(&sbi->total_hit_ext, 0); - atomic64_set(&sbi->read_hit_rbtree, 0); + /* general extent cache stats */ + for (i = 0; i < NR_EXTENT_CACHES; i++) { + atomic64_set(&sbi->total_hit_ext[i], 0); + atomic64_set(&sbi->read_hit_rbtree[i], 0); + atomic64_set(&sbi->read_hit_cached[i], 0); + } + + /* read extent_cache only */ atomic64_set(&sbi->read_hit_largest, 0); - atomic64_set(&sbi->read_hit_cached, 0); atomic_set(&sbi->inline_xattr, 0); atomic_set(&sbi->inline_inode, 0); diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index 9ee895af75e4..9936d5466b6c 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c @@ -82,7 +82,8 @@ int f2fs_init_casefolded_name(const struct inode *dir, #ifdef CONFIG_UNICODE struct super_block *sb = dir->i_sb; - if (IS_CASEFOLDED(dir)) { + if (IS_CASEFOLDED(dir) && + !is_dot_dotdot(fname->usr_fname->name, fname->usr_fname->len)) { fname->cf_name.name = kmem_cache_alloc(f2fs_cf_name_slab, GFP_NOFS); if (!fname->cf_name.name) diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c index bd16c78b5bf2..8306fcdda510 100644 --- a/fs/f2fs/extent_cache.c +++ b/fs/f2fs/extent_cache.c @@ -6,6 +6,10 @@ * Copyright (c) 2015 Samsung Electronics * Authors: Jaegeuk Kim * Chao Yu + * + * block_age-based extent cache added by: + * Copyright (c) 2022 xiaomi Co., Ltd. + * http://www.xiaomi.com/ */ #include @@ -15,6 +19,123 @@ #include "node.h" #include +static void __set_extent_info(struct extent_info *ei, + unsigned int fofs, unsigned int len, + block_t blk, bool keep_clen, + unsigned long age, unsigned long last_blocks, + enum extent_type type) +{ + ei->fofs = fofs; + ei->len = len; + + if (type == EX_READ) { + ei->blk = blk; + if (keep_clen) + return; +#ifdef CONFIG_F2FS_FS_COMPRESSION + ei->c_len = 0; +#endif + } else if (type == EX_BLOCK_AGE) { + ei->age = age; + ei->last_blocks = last_blocks; + } +} + +static bool __may_read_extent_tree(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (!test_opt(sbi, READ_EXTENT_CACHE)) + return false; + if (is_inode_flag_set(inode, FI_NO_EXTENT)) + return false; + if (is_inode_flag_set(inode, FI_COMPRESSED_FILE) && + !f2fs_sb_has_readonly(sbi)) + return false; + return S_ISREG(inode->i_mode); +} + +static bool __may_age_extent_tree(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (!test_opt(sbi, AGE_EXTENT_CACHE)) + return false; + /* don't cache block age info for cold file */ + if (is_inode_flag_set(inode, FI_COMPRESSED_FILE)) + return false; + if (file_is_cold(inode)) + return false; + + return S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode); +} + +static bool __init_may_extent_tree(struct inode *inode, enum extent_type type) +{ + if (type == EX_READ) + return __may_read_extent_tree(inode); + else if (type == EX_BLOCK_AGE) + return __may_age_extent_tree(inode); + return false; +} + +static bool __may_extent_tree(struct inode *inode, enum extent_type type) +{ + /* + * for recovered files during mount do not create extents + * if shrinker is not registered. + */ + if (list_empty(&F2FS_I_SB(inode)->s_list)) + return false; + + return __init_may_extent_tree(inode, type); +} + +static void __try_update_largest_extent(struct extent_tree *et, + struct extent_node *en) +{ + if (et->type != EX_READ) + return; + if (en->ei.len <= et->largest.len) + return; + + et->largest = en->ei; + et->largest_updated = true; +} + +static bool __is_extent_mergeable(struct extent_info *back, + struct extent_info *front, enum extent_type type) +{ + if (type == EX_READ) { +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (back->c_len && back->len != back->c_len) + return false; + if (front->c_len && front->len != front->c_len) + return false; +#endif + return (back->fofs + back->len == front->fofs && + back->blk + back->len == front->blk); + } else if (type == EX_BLOCK_AGE) { + return (back->fofs + back->len == front->fofs && + abs(back->age - front->age) <= SAME_AGE_REGION && + abs(back->last_blocks - front->last_blocks) <= + SAME_AGE_REGION); + } + return false; +} + +static bool __is_back_mergeable(struct extent_info *cur, + struct extent_info *back, enum extent_type type) +{ + return __is_extent_mergeable(back, cur, type); +} + +static bool __is_front_mergeable(struct extent_info *cur, + struct extent_info *front, enum extent_type type) +{ + return __is_extent_mergeable(cur, front, type); +} + static struct rb_entry *__lookup_rb_tree_fast(struct rb_entry *cached_re, unsigned int ofs) { @@ -237,6 +358,7 @@ static struct extent_node *__attach_extent_node(struct f2fs_sb_info *sbi, struct rb_node *parent, struct rb_node **p, bool leftmost) { + struct extent_tree_info *eti = &sbi->extent_tree[et->type]; struct extent_node *en; en = kmem_cache_alloc(extent_node_slab, GFP_ATOMIC); @@ -250,16 +372,18 @@ static struct extent_node *__attach_extent_node(struct f2fs_sb_info *sbi, rb_link_node(&en->rb_node, parent, p); rb_insert_color_cached(&en->rb_node, &et->root, leftmost); atomic_inc(&et->node_cnt); - atomic_inc(&sbi->total_ext_node); + atomic_inc(&eti->total_ext_node); return en; } static void __detach_extent_node(struct f2fs_sb_info *sbi, struct extent_tree *et, struct extent_node *en) { + struct extent_tree_info *eti = &sbi->extent_tree[et->type]; + rb_erase_cached(&en->rb_node, &et->root); atomic_dec(&et->node_cnt); - atomic_dec(&sbi->total_ext_node); + atomic_dec(&eti->total_ext_node); if (et->cached_en == en) et->cached_en = NULL; @@ -275,60 +399,50 @@ static void __detach_extent_node(struct f2fs_sb_info *sbi, static void __release_extent_node(struct f2fs_sb_info *sbi, struct extent_tree *et, struct extent_node *en) { - spin_lock(&sbi->extent_lock); + struct extent_tree_info *eti = &sbi->extent_tree[et->type]; + + spin_lock(&eti->extent_lock); f2fs_bug_on(sbi, list_empty(&en->list)); list_del_init(&en->list); - spin_unlock(&sbi->extent_lock); + spin_unlock(&eti->extent_lock); __detach_extent_node(sbi, et, en); } -static struct extent_tree *__grab_extent_tree(struct inode *inode) +static struct extent_tree *__grab_extent_tree(struct inode *inode, + enum extent_type type) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree_info *eti = &sbi->extent_tree[type]; struct extent_tree *et; nid_t ino = inode->i_ino; - mutex_lock(&sbi->extent_tree_lock); - et = radix_tree_lookup(&sbi->extent_tree_root, ino); + mutex_lock(&eti->extent_tree_lock); + et = radix_tree_lookup(&eti->extent_tree_root, ino); if (!et) { et = f2fs_kmem_cache_alloc(extent_tree_slab, GFP_NOFS); - f2fs_radix_tree_insert(&sbi->extent_tree_root, ino, et); + f2fs_radix_tree_insert(&eti->extent_tree_root, ino, et); memset(et, 0, sizeof(struct extent_tree)); et->ino = ino; + et->type = type; et->root = RB_ROOT_CACHED; et->cached_en = NULL; rwlock_init(&et->lock); INIT_LIST_HEAD(&et->list); atomic_set(&et->node_cnt, 0); - atomic_inc(&sbi->total_ext_tree); + atomic_inc(&eti->total_ext_tree); } else { - atomic_dec(&sbi->total_zombie_tree); + atomic_dec(&eti->total_zombie_tree); list_del_init(&et->list); } - mutex_unlock(&sbi->extent_tree_lock); + mutex_unlock(&eti->extent_tree_lock); /* never died until evict_inode */ - F2FS_I(inode)->extent_tree = et; + F2FS_I(inode)->extent_tree[type] = et; return et; } -static struct extent_node *__init_extent_tree(struct f2fs_sb_info *sbi, - struct extent_tree *et, struct extent_info *ei) -{ - struct rb_node **p = &et->root.rb_root.rb_node; - struct extent_node *en; - - en = __attach_extent_node(sbi, et, ei, NULL, p, true); - if (!en) - return NULL; - - et->largest = en->ei; - et->cached_en = en; - return en; -} - static unsigned int __free_extent_tree(struct f2fs_sb_info *sbi, struct extent_tree *et) { @@ -357,70 +471,89 @@ static void __drop_largest_extent(struct extent_tree *et, } } -/* return true, if inode page is changed */ -static void __f2fs_init_extent_tree(struct inode *inode, struct page *ipage) +void f2fs_init_read_extent_tree(struct inode *inode, struct page *ipage) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct f2fs_extent *i_ext = ipage ? &F2FS_INODE(ipage)->i_ext : NULL; + struct extent_tree_info *eti = &sbi->extent_tree[EX_READ]; + struct f2fs_extent *i_ext = &F2FS_INODE(ipage)->i_ext; struct extent_tree *et; struct extent_node *en; struct extent_info ei; - if (!f2fs_may_extent_tree(inode)) { - /* drop largest extent */ + if (!__may_extent_tree(inode, EX_READ)) { + /* drop largest read extent */ if (i_ext && i_ext->len) { f2fs_wait_on_page_writeback(ipage, NODE, true, true); i_ext->len = 0; set_page_dirty(ipage); - return; } - return; + goto out; } - et = __grab_extent_tree(inode); + et = __grab_extent_tree(inode, EX_READ); if (!i_ext || !i_ext->len) - return; + goto out; - get_extent_info(&ei, i_ext); + get_read_extent_info(&ei, i_ext); write_lock(&et->lock); if (atomic_read(&et->node_cnt)) - goto out; + goto unlock_out; - en = __init_extent_tree(sbi, et, &ei); + en = __attach_extent_node(sbi, et, &ei, NULL, + &et->root.rb_root.rb_node, true); if (en) { - spin_lock(&sbi->extent_lock); - list_add_tail(&en->list, &sbi->extent_list); - spin_unlock(&sbi->extent_lock); + et->largest = en->ei; + et->cached_en = en; + + spin_lock(&eti->extent_lock); + list_add_tail(&en->list, &eti->extent_list); + spin_unlock(&eti->extent_lock); } -out: +unlock_out: write_unlock(&et->lock); -} - -void f2fs_init_extent_tree(struct inode *inode, struct page *ipage) -{ - __f2fs_init_extent_tree(inode, ipage); - - if (!F2FS_I(inode)->extent_tree) +out: + if (!F2FS_I(inode)->extent_tree[EX_READ]) set_inode_flag(inode, FI_NO_EXTENT); } -static bool f2fs_lookup_extent_tree(struct inode *inode, pgoff_t pgofs, - struct extent_info *ei) +void f2fs_init_age_extent_tree(struct inode *inode) +{ + if (!__init_may_extent_tree(inode, EX_BLOCK_AGE)) + return; + __grab_extent_tree(inode, EX_BLOCK_AGE); +} + +void f2fs_init_extent_tree(struct inode *inode) +{ + /* initialize read cache */ + if (__init_may_extent_tree(inode, EX_READ)) + __grab_extent_tree(inode, EX_READ); + + /* initialize block age cache */ + if (__init_may_extent_tree(inode, EX_BLOCK_AGE)) + __grab_extent_tree(inode, EX_BLOCK_AGE); +} + +static bool __lookup_extent_tree(struct inode *inode, pgoff_t pgofs, + struct extent_info *ei, enum extent_type type) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct extent_tree *et = F2FS_I(inode)->extent_tree; + struct extent_tree_info *eti = &sbi->extent_tree[type]; + struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; struct extent_node *en; bool ret = false; - f2fs_bug_on(sbi, !et); + if (!et) + return false; - trace_f2fs_lookup_extent_tree_start(inode, pgofs); + trace_f2fs_lookup_extent_tree_start(inode, pgofs, type); read_lock(&et->lock); - if (et->largest.fofs <= pgofs && + if (type == EX_READ && + et->largest.fofs <= pgofs && et->largest.fofs + et->largest.len > pgofs) { *ei = et->largest; ret = true; @@ -434,23 +567,26 @@ static bool f2fs_lookup_extent_tree(struct inode *inode, pgoff_t pgofs, goto out; if (en == et->cached_en) - stat_inc_cached_node_hit(sbi); + stat_inc_cached_node_hit(sbi, type); else - stat_inc_rbtree_node_hit(sbi); + stat_inc_rbtree_node_hit(sbi, type); *ei = en->ei; - spin_lock(&sbi->extent_lock); + spin_lock(&eti->extent_lock); if (!list_empty(&en->list)) { - list_move_tail(&en->list, &sbi->extent_list); + list_move_tail(&en->list, &eti->extent_list); et->cached_en = en; } - spin_unlock(&sbi->extent_lock); + spin_unlock(&eti->extent_lock); ret = true; out: - stat_inc_total_hit(sbi); + stat_inc_total_hit(sbi, type); read_unlock(&et->lock); - trace_f2fs_lookup_extent_tree_end(inode, pgofs, ei); + if (type == EX_READ) + trace_f2fs_lookup_read_extent_tree_end(inode, pgofs, ei); + else if (type == EX_BLOCK_AGE) + trace_f2fs_lookup_age_extent_tree_end(inode, pgofs, ei); return ret; } @@ -459,18 +595,20 @@ static struct extent_node *__try_merge_extent_node(struct f2fs_sb_info *sbi, struct extent_node *prev_ex, struct extent_node *next_ex) { + struct extent_tree_info *eti = &sbi->extent_tree[et->type]; struct extent_node *en = NULL; - if (prev_ex && __is_back_mergeable(ei, &prev_ex->ei)) { + if (prev_ex && __is_back_mergeable(ei, &prev_ex->ei, et->type)) { prev_ex->ei.len += ei->len; ei = &prev_ex->ei; en = prev_ex; } - if (next_ex && __is_front_mergeable(ei, &next_ex->ei)) { + if (next_ex && __is_front_mergeable(ei, &next_ex->ei, et->type)) { next_ex->ei.fofs = ei->fofs; - next_ex->ei.blk = ei->blk; next_ex->ei.len += ei->len; + if (et->type == EX_READ) + next_ex->ei.blk = ei->blk; if (en) __release_extent_node(sbi, et, prev_ex); @@ -482,12 +620,12 @@ static struct extent_node *__try_merge_extent_node(struct f2fs_sb_info *sbi, __try_update_largest_extent(et, en); - spin_lock(&sbi->extent_lock); + spin_lock(&eti->extent_lock); if (!list_empty(&en->list)) { - list_move_tail(&en->list, &sbi->extent_list); + list_move_tail(&en->list, &eti->extent_list); et->cached_en = en; } - spin_unlock(&sbi->extent_lock); + spin_unlock(&eti->extent_lock); return en; } @@ -497,6 +635,7 @@ static struct extent_node *__insert_extent_tree(struct f2fs_sb_info *sbi, struct rb_node *insert_parent, bool leftmost) { + struct extent_tree_info *eti = &sbi->extent_tree[et->type]; struct rb_node **p; struct rb_node *parent = NULL; struct extent_node *en = NULL; @@ -519,48 +658,55 @@ do_insert: __try_update_largest_extent(et, en); /* update in global extent list */ - spin_lock(&sbi->extent_lock); - list_add_tail(&en->list, &sbi->extent_list); + spin_lock(&eti->extent_lock); + list_add_tail(&en->list, &eti->extent_list); et->cached_en = en; - spin_unlock(&sbi->extent_lock); + spin_unlock(&eti->extent_lock); return en; } -static void f2fs_update_extent_tree_range(struct inode *inode, - pgoff_t fofs, block_t blkaddr, unsigned int len) +static void __update_extent_tree_range(struct inode *inode, + struct extent_info *tei, enum extent_type type) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct extent_tree *et = F2FS_I(inode)->extent_tree; + struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; struct extent_node *en = NULL, *en1 = NULL; struct extent_node *prev_en = NULL, *next_en = NULL; struct extent_info ei, dei, prev; struct rb_node **insert_p = NULL, *insert_parent = NULL; + unsigned int fofs = tei->fofs, len = tei->len; unsigned int end = fofs + len; - unsigned int pos = (unsigned int)fofs; bool updated = false; bool leftmost = false; if (!et) return; - trace_f2fs_update_extent_tree_range(inode, fofs, blkaddr, len); + if (type == EX_READ) + trace_f2fs_update_read_extent_tree_range(inode, fofs, len, + tei->blk, 0); + else if (type == EX_BLOCK_AGE) + trace_f2fs_update_age_extent_tree_range(inode, fofs, len, + tei->age, tei->last_blocks); write_lock(&et->lock); - if (is_inode_flag_set(inode, FI_NO_EXTENT)) { - write_unlock(&et->lock); - return; + if (type == EX_READ) { + if (is_inode_flag_set(inode, FI_NO_EXTENT)) { + write_unlock(&et->lock); + return; + } + + prev = et->largest; + dei.len = 0; + + /* + * drop largest extent before lookup, in case it's already + * been shrunk from extent tree + */ + __drop_largest_extent(et, fofs, len); } - prev = et->largest; - dei.len = 0; - - /* - * drop largest extent before lookup, in case it's already - * been shrunk from extent tree - */ - __drop_largest_extent(et, fofs, len); - /* 1. lookup first extent node in range [fofs, fofs + len - 1] */ en = (struct extent_node *)f2fs_lookup_rb_tree_ret(&et->root, (struct rb_entry *)et->cached_en, fofs, @@ -580,26 +726,32 @@ static void f2fs_update_extent_tree_range(struct inode *inode, dei = en->ei; org_end = dei.fofs + dei.len; - f2fs_bug_on(sbi, pos >= org_end); + f2fs_bug_on(sbi, fofs >= org_end); - if (pos > dei.fofs && pos - dei.fofs >= F2FS_MIN_EXTENT_LEN) { - en->ei.len = pos - en->ei.fofs; + if (fofs > dei.fofs && (type != EX_READ || + fofs - dei.fofs >= F2FS_MIN_EXTENT_LEN)) { + en->ei.len = fofs - en->ei.fofs; prev_en = en; parts = 1; } - if (end < org_end && org_end - end >= F2FS_MIN_EXTENT_LEN) { + if (end < org_end && (type != EX_READ || + org_end - end >= F2FS_MIN_EXTENT_LEN)) { if (parts) { - set_extent_info(&ei, end, - end - dei.fofs + dei.blk, - org_end - end); + __set_extent_info(&ei, + end, org_end - end, + end - dei.fofs + dei.blk, false, + dei.age, dei.last_blocks, + type); en1 = __insert_extent_tree(sbi, et, &ei, NULL, NULL, true); next_en = en1; } else { - en->ei.fofs = end; - en->ei.blk += end - dei.fofs; - en->ei.len -= end - dei.fofs; + __set_extent_info(&en->ei, + end, en->ei.len - (end - dei.fofs), + en->ei.blk + (end - dei.fofs), true, + dei.age, dei.last_blocks, + type); next_en = en; } parts++; @@ -629,10 +781,15 @@ static void f2fs_update_extent_tree_range(struct inode *inode, en = next_en; } - /* 3. update extent in extent cache */ - if (blkaddr) { + if (type == EX_BLOCK_AGE) + goto update_age_extent_cache; - set_extent_info(&ei, fofs, blkaddr, len); + /* 3. update extent in read extent cache */ + BUG_ON(type != EX_READ); + + if (tei->blk) { + __set_extent_info(&ei, fofs, len, tei->blk, false, + 0, 0, EX_READ); if (!__try_merge_extent_node(sbi, et, &ei, prev_en, next_en)) __insert_extent_tree(sbi, et, &ei, insert_p, insert_parent, leftmost); @@ -654,31 +811,173 @@ static void f2fs_update_extent_tree_range(struct inode *inode, et->largest_updated = false; updated = true; } + goto out_read_extent_cache; +update_age_extent_cache: + if (!tei->last_blocks) + goto out_read_extent_cache; + __set_extent_info(&ei, fofs, len, 0, false, + tei->age, tei->last_blocks, EX_BLOCK_AGE); + if (!__try_merge_extent_node(sbi, et, &ei, prev_en, next_en)) + __insert_extent_tree(sbi, et, &ei, + insert_p, insert_parent, leftmost); +out_read_extent_cache: write_unlock(&et->lock); if (updated) f2fs_mark_inode_dirty_sync(inode, true); } -unsigned int f2fs_shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink) +#ifdef CONFIG_F2FS_FS_COMPRESSION +void f2fs_update_read_extent_tree_range_compressed(struct inode *inode, + pgoff_t fofs, block_t blkaddr, unsigned int llen, + unsigned int c_len) { + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et = F2FS_I(inode)->extent_tree[EX_READ]; + struct extent_node *en = NULL; + struct extent_node *prev_en = NULL, *next_en = NULL; + struct extent_info ei; + struct rb_node **insert_p = NULL, *insert_parent = NULL; + bool leftmost = false; + + trace_f2fs_update_read_extent_tree_range(inode, fofs, llen, + blkaddr, c_len); + + /* it is safe here to check FI_NO_EXTENT w/o et->lock in ro image */ + if (is_inode_flag_set(inode, FI_NO_EXTENT)) + return; + + write_lock(&et->lock); + + en = (struct extent_node *)f2fs_lookup_rb_tree_ret(&et->root, + (struct rb_entry *)et->cached_en, fofs, + (struct rb_entry **)&prev_en, + (struct rb_entry **)&next_en, + &insert_p, &insert_parent, false, + &leftmost); + if (en) + goto unlock_out; + + __set_extent_info(&ei, fofs, llen, blkaddr, true, 0, 0, EX_READ); + ei.c_len = c_len; + + if (!__try_merge_extent_node(sbi, et, &ei, prev_en, next_en)) + __insert_extent_tree(sbi, et, &ei, + insert_p, insert_parent, leftmost); +unlock_out: + write_unlock(&et->lock); +} +#endif + +static unsigned long long __calculate_block_age(unsigned long long new, + unsigned long long old) +{ + unsigned long long diff; + + diff = (new >= old) ? new - (new - old) : new + (old - new); + + return div_u64(diff * LAST_AGE_WEIGHT, 100); +} + +/* This returns a new age and allocated blocks in ei */ +static int __get_new_block_age(struct inode *inode, struct extent_info *ei, + block_t blkaddr) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + loff_t f_size = i_size_read(inode); + unsigned long long cur_blocks = + atomic64_read(&sbi->allocated_data_blocks); + struct extent_info tei = *ei; /* only fofs and len are valid */ + + /* + * When I/O is not aligned to a PAGE_SIZE, update will happen to the last + * file block even in seq write. So don't record age for newly last file + * block here. + */ + if ((f_size >> PAGE_SHIFT) == ei->fofs && f_size & (PAGE_SIZE - 1) && + blkaddr == NEW_ADDR) + return -EINVAL; + + if (__lookup_extent_tree(inode, ei->fofs, &tei, EX_BLOCK_AGE)) { + unsigned long long cur_age; + + if (cur_blocks >= tei.last_blocks) + cur_age = cur_blocks - tei.last_blocks; + else + /* allocated_data_blocks overflow */ + cur_age = ULLONG_MAX - tei.last_blocks + cur_blocks; + + if (tei.age) + ei->age = __calculate_block_age(cur_age, tei.age); + else + ei->age = cur_age; + ei->last_blocks = cur_blocks; + WARN_ON(ei->age > cur_blocks); + return 0; + } + + f2fs_bug_on(sbi, blkaddr == NULL_ADDR); + + /* the data block was allocated for the first time */ + if (blkaddr == NEW_ADDR) + goto out; + + if (__is_valid_data_blkaddr(blkaddr) && + !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) { + f2fs_bug_on(sbi, 1); + return -EINVAL; + } +out: + /* + * init block age with zero, this can happen when the block age extent + * was reclaimed due to memory constraint or system reboot + */ + ei->age = 0; + ei->last_blocks = cur_blocks; + return 0; +} + +static void __update_extent_cache(struct dnode_of_data *dn, enum extent_type type) +{ + struct extent_info ei = {}; + + if (!__may_extent_tree(dn->inode, type)) + return; + + ei.fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) + + dn->ofs_in_node; + ei.len = 1; + + if (type == EX_READ) { + if (dn->data_blkaddr == NEW_ADDR) + ei.blk = NULL_ADDR; + else + ei.blk = dn->data_blkaddr; + } else if (type == EX_BLOCK_AGE) { + if (__get_new_block_age(dn->inode, &ei, dn->data_blkaddr)) + return; + } + __update_extent_tree_range(dn->inode, &ei, type); +} + +static unsigned int __shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink, + enum extent_type type) +{ + struct extent_tree_info *eti = &sbi->extent_tree[type]; struct extent_tree *et, *next; struct extent_node *en; unsigned int node_cnt = 0, tree_cnt = 0; int remained; - if (!test_opt(sbi, EXTENT_CACHE)) - return 0; - - if (!atomic_read(&sbi->total_zombie_tree)) + if (!atomic_read(&eti->total_zombie_tree)) goto free_node; - if (!mutex_trylock(&sbi->extent_tree_lock)) + if (!mutex_trylock(&eti->extent_tree_lock)) goto out; /* 1. remove unreferenced extent tree */ - list_for_each_entry_safe(et, next, &sbi->zombie_list, list) { + list_for_each_entry_safe(et, next, &eti->zombie_list, list) { if (atomic_read(&et->node_cnt)) { write_lock(&et->lock); node_cnt += __free_extent_tree(sbi, et); @@ -686,61 +985,137 @@ unsigned int f2fs_shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink) } f2fs_bug_on(sbi, atomic_read(&et->node_cnt)); list_del_init(&et->list); - radix_tree_delete(&sbi->extent_tree_root, et->ino); + radix_tree_delete(&eti->extent_tree_root, et->ino); kmem_cache_free(extent_tree_slab, et); - atomic_dec(&sbi->total_ext_tree); - atomic_dec(&sbi->total_zombie_tree); + atomic_dec(&eti->total_ext_tree); + atomic_dec(&eti->total_zombie_tree); tree_cnt++; if (node_cnt + tree_cnt >= nr_shrink) goto unlock_out; cond_resched(); } - mutex_unlock(&sbi->extent_tree_lock); + mutex_unlock(&eti->extent_tree_lock); free_node: /* 2. remove LRU extent entries */ - if (!mutex_trylock(&sbi->extent_tree_lock)) + if (!mutex_trylock(&eti->extent_tree_lock)) goto out; remained = nr_shrink - (node_cnt + tree_cnt); - spin_lock(&sbi->extent_lock); + spin_lock(&eti->extent_lock); for (; remained > 0; remained--) { - if (list_empty(&sbi->extent_list)) + if (list_empty(&eti->extent_list)) break; - en = list_first_entry(&sbi->extent_list, + en = list_first_entry(&eti->extent_list, struct extent_node, list); et = en->et; if (!write_trylock(&et->lock)) { /* refresh this extent node's position in extent list */ - list_move_tail(&en->list, &sbi->extent_list); + list_move_tail(&en->list, &eti->extent_list); continue; } list_del_init(&en->list); - spin_unlock(&sbi->extent_lock); + spin_unlock(&eti->extent_lock); __detach_extent_node(sbi, et, en); write_unlock(&et->lock); node_cnt++; - spin_lock(&sbi->extent_lock); + spin_lock(&eti->extent_lock); } - spin_unlock(&sbi->extent_lock); + spin_unlock(&eti->extent_lock); unlock_out: - mutex_unlock(&sbi->extent_tree_lock); + mutex_unlock(&eti->extent_tree_lock); out: - trace_f2fs_shrink_extent_tree(sbi, node_cnt, tree_cnt); + trace_f2fs_shrink_extent_tree(sbi, node_cnt, tree_cnt, type); return node_cnt + tree_cnt; } -unsigned int f2fs_destroy_extent_node(struct inode *inode) +/* read extent cache operations */ +bool f2fs_lookup_read_extent_cache(struct inode *inode, pgoff_t pgofs, + struct extent_info *ei) +{ + if (!__may_extent_tree(inode, EX_READ)) + return false; + + return __lookup_extent_tree(inode, pgofs, ei, EX_READ); +} + +void f2fs_update_read_extent_cache(struct dnode_of_data *dn) +{ + return __update_extent_cache(dn, EX_READ); +} + +void f2fs_update_read_extent_cache_range(struct dnode_of_data *dn, + pgoff_t fofs, block_t blkaddr, unsigned int len) +{ + struct extent_info ei = { + .fofs = fofs, + .len = len, + .blk = blkaddr, + }; + + if (!__may_extent_tree(dn->inode, EX_READ)) + return; + + __update_extent_tree_range(dn->inode, &ei, EX_READ); +} + +unsigned int f2fs_shrink_read_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink) +{ + if (!test_opt(sbi, READ_EXTENT_CACHE)) + return 0; + + return __shrink_extent_tree(sbi, nr_shrink, EX_READ); +} + +/* block age extent cache operations */ +bool f2fs_lookup_age_extent_cache(struct inode *inode, pgoff_t pgofs, + struct extent_info *ei) +{ + if (!__may_extent_tree(inode, EX_BLOCK_AGE)) + return false; + + return __lookup_extent_tree(inode, pgofs, ei, EX_BLOCK_AGE); +} + +void f2fs_update_age_extent_cache(struct dnode_of_data *dn) +{ + return __update_extent_cache(dn, EX_BLOCK_AGE); +} + +void f2fs_update_age_extent_cache_range(struct dnode_of_data *dn, + pgoff_t fofs, unsigned int len) +{ + struct extent_info ei = { + .fofs = fofs, + .len = len, + }; + + if (!__may_extent_tree(dn->inode, EX_BLOCK_AGE)) + return; + + __update_extent_tree_range(dn->inode, &ei, EX_BLOCK_AGE); +} + +unsigned int f2fs_shrink_age_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink) +{ + if (!test_opt(sbi, AGE_EXTENT_CACHE)) + return 0; + + return __shrink_extent_tree(sbi, nr_shrink, EX_BLOCK_AGE); +} + +static unsigned int __destroy_extent_node(struct inode *inode, + enum extent_type type) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct extent_tree *et = F2FS_I(inode)->extent_tree; + struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; unsigned int node_cnt = 0; if (!et || !atomic_read(&et->node_cnt)) @@ -753,31 +1128,47 @@ unsigned int f2fs_destroy_extent_node(struct inode *inode) return node_cnt; } -void f2fs_drop_extent_tree(struct inode *inode) +void f2fs_destroy_extent_node(struct inode *inode) +{ + __destroy_extent_node(inode, EX_READ); + __destroy_extent_node(inode, EX_BLOCK_AGE); +} + +static void __drop_extent_tree(struct inode *inode, enum extent_type type) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct extent_tree *et = F2FS_I(inode)->extent_tree; + struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; bool updated = false; - if (!f2fs_may_extent_tree(inode)) + if (!__may_extent_tree(inode, type)) return; write_lock(&et->lock); set_inode_flag(inode, FI_NO_EXTENT); __free_extent_tree(sbi, et); - if (et->largest.len) { - et->largest.len = 0; - updated = true; + if (type == EX_READ) { + set_inode_flag(inode, FI_NO_EXTENT); + if (et->largest.len) { + et->largest.len = 0; + updated = true; + } } write_unlock(&et->lock); if (updated) f2fs_mark_inode_dirty_sync(inode, true); } -void f2fs_destroy_extent_tree(struct inode *inode) +void f2fs_drop_extent_tree(struct inode *inode) +{ + __drop_extent_tree(inode, EX_READ); + __drop_extent_tree(inode, EX_BLOCK_AGE); +} + +static void __destroy_extent_tree(struct inode *inode, enum extent_type type) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct extent_tree *et = F2FS_I(inode)->extent_tree; + struct extent_tree_info *eti = &sbi->extent_tree[type]; + struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; unsigned int node_cnt = 0; if (!et) @@ -785,76 +1176,56 @@ void f2fs_destroy_extent_tree(struct inode *inode) if (inode->i_nlink && !is_bad_inode(inode) && atomic_read(&et->node_cnt)) { - mutex_lock(&sbi->extent_tree_lock); - list_add_tail(&et->list, &sbi->zombie_list); - atomic_inc(&sbi->total_zombie_tree); - mutex_unlock(&sbi->extent_tree_lock); + mutex_lock(&eti->extent_tree_lock); + list_add_tail(&et->list, &eti->zombie_list); + atomic_inc(&eti->total_zombie_tree); + mutex_unlock(&eti->extent_tree_lock); return; } /* free all extent info belong to this extent tree */ - node_cnt = f2fs_destroy_extent_node(inode); + node_cnt = __destroy_extent_node(inode, type); /* delete extent tree entry in radix tree */ - mutex_lock(&sbi->extent_tree_lock); + mutex_lock(&eti->extent_tree_lock); f2fs_bug_on(sbi, atomic_read(&et->node_cnt)); - radix_tree_delete(&sbi->extent_tree_root, inode->i_ino); + radix_tree_delete(&eti->extent_tree_root, inode->i_ino); kmem_cache_free(extent_tree_slab, et); - atomic_dec(&sbi->total_ext_tree); - mutex_unlock(&sbi->extent_tree_lock); + atomic_dec(&eti->total_ext_tree); + mutex_unlock(&eti->extent_tree_lock); - F2FS_I(inode)->extent_tree = NULL; + F2FS_I(inode)->extent_tree[type] = NULL; - trace_f2fs_destroy_extent_tree(inode, node_cnt); + trace_f2fs_destroy_extent_tree(inode, node_cnt, type); } -bool f2fs_lookup_extent_cache(struct inode *inode, pgoff_t pgofs, - struct extent_info *ei) +void f2fs_destroy_extent_tree(struct inode *inode) { - if (!f2fs_may_extent_tree(inode)) - return false; - - return f2fs_lookup_extent_tree(inode, pgofs, ei); + __destroy_extent_tree(inode, EX_READ); + __destroy_extent_tree(inode, EX_BLOCK_AGE); } -void f2fs_update_extent_cache(struct dnode_of_data *dn) +static void __init_extent_tree_info(struct extent_tree_info *eti) { - pgoff_t fofs; - block_t blkaddr; - - if (!f2fs_may_extent_tree(dn->inode)) - return; - - if (dn->data_blkaddr == NEW_ADDR) - blkaddr = NULL_ADDR; - else - blkaddr = dn->data_blkaddr; - - fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) + - dn->ofs_in_node; - f2fs_update_extent_tree_range(dn->inode, fofs, blkaddr, 1); -} - -void f2fs_update_extent_cache_range(struct dnode_of_data *dn, - pgoff_t fofs, block_t blkaddr, unsigned int len) - -{ - if (!f2fs_may_extent_tree(dn->inode)) - return; - - f2fs_update_extent_tree_range(dn->inode, fofs, blkaddr, len); + INIT_RADIX_TREE(&eti->extent_tree_root, GFP_NOIO); + mutex_init(&eti->extent_tree_lock); + INIT_LIST_HEAD(&eti->extent_list); + spin_lock_init(&eti->extent_lock); + atomic_set(&eti->total_ext_tree, 0); + INIT_LIST_HEAD(&eti->zombie_list); + atomic_set(&eti->total_zombie_tree, 0); + atomic_set(&eti->total_ext_node, 0); } void f2fs_init_extent_cache_info(struct f2fs_sb_info *sbi) { - INIT_RADIX_TREE(&sbi->extent_tree_root, GFP_NOIO); - mutex_init(&sbi->extent_tree_lock); - INIT_LIST_HEAD(&sbi->extent_list); - spin_lock_init(&sbi->extent_lock); - atomic_set(&sbi->total_ext_tree, 0); - INIT_LIST_HEAD(&sbi->zombie_list); - atomic_set(&sbi->total_zombie_tree, 0); - atomic_set(&sbi->total_ext_node, 0); + __init_extent_tree_info(&sbi->extent_tree[EX_READ]); + __init_extent_tree_info(&sbi->extent_tree[EX_BLOCK_AGE]); + + /* initialize for block age extents */ + atomic64_set(&sbi->allocated_data_blocks, 0); + sbi->hot_data_age_threshold = DEF_HOT_DATA_AGE_THRESHOLD; + sbi->warm_data_age_threshold = DEF_WARM_DATA_AGE_THRESHOLD; } int __init f2fs_create_extent_cache(void) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index e20fd009987d..86abbb1a6f1d 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -84,7 +84,7 @@ extern const char *f2fs_fault_name[FAULT_MAX]; #define F2FS_MOUNT_FLUSH_MERGE 0x00000400 #define F2FS_MOUNT_NOBARRIER 0x00000800 #define F2FS_MOUNT_FASTBOOT 0x00001000 -#define F2FS_MOUNT_EXTENT_CACHE 0x00002000 +#define F2FS_MOUNT_READ_EXTENT_CACHE 0x00002000 #define F2FS_MOUNT_DATA_FLUSH 0x00008000 #define F2FS_MOUNT_FAULT_INJECTION 0x00010000 #define F2FS_MOUNT_USRQUOTA 0x00080000 @@ -99,6 +99,7 @@ extern const char *f2fs_fault_name[FAULT_MAX]; #define F2FS_MOUNT_MERGE_CHECKPOINT 0x10000000 #define F2FS_MOUNT_GC_MERGE 0x20000000 #define F2FS_MOUNT_COMPRESS_CACHE 0x40000000 +#define F2FS_MOUNT_AGE_EXTENT_CACHE 0x80000000 #define F2FS_OPTION(sbi) ((sbi)->mount_opt) #define clear_opt(sbi, option) (F2FS_OPTION(sbi).opt &= ~F2FS_MOUNT_##option) @@ -494,11 +495,11 @@ struct f2fs_filename { #ifdef CONFIG_UNICODE /* * For casefolded directories: the casefolded name, but it's left NULL - * if the original name is not valid Unicode, if the directory is both - * casefolded and encrypted and its encryption key is unavailable, or if - * the filesystem is doing an internal operation where usr_fname is also - * NULL. In all these cases we fall back to treating the name as an - * opaque byte sequence. + * if the original name is not valid Unicode, if the original name is + * "." or "..", if the directory is both casefolded and encrypted and + * its encryption key is unavailable, or if the filesystem is doing an + * internal operation where usr_fname is also NULL. In all these cases + * we fall back to treating the name as an opaque byte sequence. */ struct fscrypt_str cf_name; #endif @@ -572,7 +573,26 @@ enum { #define F2FS_MIN_EXTENT_LEN 64 /* minimum extent length */ /* number of extent info in extent cache we try to shrink */ -#define EXTENT_CACHE_SHRINK_NUMBER 128 +#define READ_EXTENT_CACHE_SHRINK_NUMBER 128 + +/* number of age extent info in extent cache we try to shrink */ +#define AGE_EXTENT_CACHE_SHRINK_NUMBER 128 +#define LAST_AGE_WEIGHT 30 +#define SAME_AGE_REGION 1024 + +/* + * Define data block with age less than 1GB as hot data + * define data block with age less than 10GB but more than 1GB as warm data + */ +#define DEF_HOT_DATA_AGE_THRESHOLD 262144 +#define DEF_WARM_DATA_AGE_THRESHOLD 2621440 + +/* extent cache type */ +enum extent_type { + EX_READ, + EX_BLOCK_AGE, + NR_EXTENT_CACHES, +}; struct rb_entry { struct rb_node rb_node; /* rb node located in rb-tree */ @@ -588,7 +608,24 @@ struct rb_entry { struct extent_info { unsigned int fofs; /* start offset in a file */ unsigned int len; /* length of the extent */ - u32 blk; /* start block address of the extent */ + union { + /* read extent_cache */ + struct { + /* start block address of the extent */ + block_t blk; +#ifdef CONFIG_F2FS_FS_COMPRESSION + /* physical extent length of compressed blocks */ + unsigned int c_len; +#endif + }; + /* block age extent_cache */ + struct { + /* block age of the extent */ + unsigned long long age; + /* last total blocks allocated */ + unsigned long long last_blocks; + }; + }; }; struct extent_node { @@ -600,13 +637,25 @@ struct extent_node { struct extent_tree { nid_t ino; /* inode number */ + enum extent_type type; /* keep the extent tree type */ struct rb_root_cached root; /* root of extent info rb-tree */ struct extent_node *cached_en; /* recently accessed extent node */ - struct extent_info largest; /* largested extent info */ struct list_head list; /* to be used by sbi->zombie_list */ rwlock_t lock; /* protect extent info rb-tree */ atomic_t node_cnt; /* # of extent node in rb-tree*/ bool largest_updated; /* largest extent updated */ + struct extent_info largest; /* largest cached extent for EX_READ */ +}; + +struct extent_tree_info { + struct radix_tree_root extent_tree_root;/* cache extent cache entries */ + struct mutex extent_tree_lock; /* locking extent radix tree */ + struct list_head extent_list; /* lru list for shrinker */ + spinlock_t extent_lock; /* locking extent lru list */ + atomic_t total_ext_tree; /* extent tree count */ + struct list_head zombie_list; /* extent zombie tree list */ + atomic_t total_zombie_tree; /* extent zombie tree count */ + atomic_t total_ext_node; /* extent info count */ }; /* @@ -766,7 +815,8 @@ struct f2fs_inode_info { struct list_head inmem_pages; /* inmemory pages managed by f2fs */ struct task_struct *inmem_task; /* store inmemory task */ struct mutex inmem_lock; /* lock for inmemory pages */ - struct extent_tree *extent_tree; /* cached extent_tree entry */ + struct extent_tree *extent_tree[NR_EXTENT_CACHES]; + /* cached extent_tree entry */ /* avoid racing between foreground op and gc */ struct f2fs_rwsem i_gc_rwsem[2]; @@ -788,7 +838,7 @@ struct f2fs_inode_info { unsigned int i_cluster_size; /* cluster size */ }; -static inline void get_extent_info(struct extent_info *ext, +static inline void get_read_extent_info(struct extent_info *ext, struct f2fs_extent *i_ext) { ext->fofs = le32_to_cpu(i_ext->fofs); @@ -796,7 +846,7 @@ static inline void get_extent_info(struct extent_info *ext, ext->len = le32_to_cpu(i_ext->len); } -static inline void set_raw_extent(struct extent_info *ext, +static inline void set_raw_read_extent(struct extent_info *ext, struct f2fs_extent *i_ext) { i_ext->fofs = cpu_to_le32(ext->fofs); @@ -804,14 +854,6 @@ static inline void set_raw_extent(struct extent_info *ext, i_ext->len = cpu_to_le32(ext->len); } -static inline void set_extent_info(struct extent_info *ei, unsigned int fofs, - u32 blk, unsigned int len) -{ - ei->fofs = fofs; - ei->blk = blk; - ei->len = len; -} - static inline bool __is_discard_mergeable(struct discard_info *back, struct discard_info *front, unsigned int max_len) { @@ -831,35 +873,6 @@ static inline bool __is_discard_front_mergeable(struct discard_info *cur, return __is_discard_mergeable(cur, front, max_len); } -static inline bool __is_extent_mergeable(struct extent_info *back, - struct extent_info *front) -{ - return (back->fofs + back->len == front->fofs && - back->blk + back->len == front->blk); -} - -static inline bool __is_back_mergeable(struct extent_info *cur, - struct extent_info *back) -{ - return __is_extent_mergeable(back, cur); -} - -static inline bool __is_front_mergeable(struct extent_info *cur, - struct extent_info *front) -{ - return __is_extent_mergeable(cur, front); -} - -extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync); -static inline void __try_update_largest_extent(struct extent_tree *et, - struct extent_node *en) -{ - if (en->ei.len > et->largest.len) { - et->largest = en->ei; - et->largest_updated = true; - } -} - /* * For free nid management */ @@ -1601,14 +1614,12 @@ struct f2fs_sb_info { struct mutex flush_lock; /* for flush exclusion */ /* for extent tree cache */ - struct radix_tree_root extent_tree_root;/* cache extent cache entries */ - struct mutex extent_tree_lock; /* locking extent radix tree */ - struct list_head extent_list; /* lru list for shrinker */ - spinlock_t extent_lock; /* locking extent lru list */ - atomic_t total_ext_tree; /* extent tree count */ - struct list_head zombie_list; /* extent zombie tree list */ - atomic_t total_zombie_tree; /* extent zombie tree count */ - atomic_t total_ext_node; /* extent info count */ + struct extent_tree_info extent_tree[NR_EXTENT_CACHES]; + atomic64_t allocated_data_blocks; /* for block age extent_cache */ + + /* The threshold used for hot and warm data seperation*/ + unsigned int hot_data_age_threshold; + unsigned int warm_data_age_threshold; /* basic filesystem units */ unsigned int log_sectors_per_block; /* log2 sectors per block */ @@ -1689,10 +1700,14 @@ struct f2fs_sb_info { unsigned int segment_count[2]; /* # of allocated segments */ unsigned int block_count[2]; /* # of allocated blocks */ atomic_t inplace_count; /* # of inplace update */ - atomic64_t total_hit_ext; /* # of lookup extent cache */ - atomic64_t read_hit_rbtree; /* # of hit rbtree extent node */ - atomic64_t read_hit_largest; /* # of hit largest extent node */ - atomic64_t read_hit_cached; /* # of hit cached extent node */ + /* # of lookup extent cache */ + atomic64_t total_hit_ext[NR_EXTENT_CACHES]; + /* # of hit rbtree extent node */ + atomic64_t read_hit_rbtree[NR_EXTENT_CACHES]; + /* # of hit cached extent node */ + atomic64_t read_hit_cached[NR_EXTENT_CACHES]; + /* # of hit largest extent node in read extent cache */ + atomic64_t read_hit_largest; atomic_t inline_xattr; /* # of inline_xattr inodes */ atomic_t inline_inode; /* # of inline_data inodes */ atomic_t inline_dir; /* # of inline_dentry inodes */ @@ -2485,6 +2500,7 @@ static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi) return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum); } +extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync); static inline int inc_valid_node_count(struct f2fs_sb_info *sbi, struct inode *inode, bool is_inode) { @@ -3775,9 +3791,19 @@ struct f2fs_stat_info { struct f2fs_sb_info *sbi; int all_area_segs, sit_area_segs, nat_area_segs, ssa_area_segs; int main_area_segs, main_area_sections, main_area_zones; - unsigned long long hit_largest, hit_cached, hit_rbtree; - unsigned long long hit_total, total_ext; - int ext_tree, zombie_tree, ext_node; + unsigned long long hit_cached[NR_EXTENT_CACHES]; + unsigned long long hit_rbtree[NR_EXTENT_CACHES]; + unsigned long long total_ext[NR_EXTENT_CACHES]; + unsigned long long hit_total[NR_EXTENT_CACHES]; + int ext_tree[NR_EXTENT_CACHES]; + int zombie_tree[NR_EXTENT_CACHES]; + int ext_node[NR_EXTENT_CACHES]; + /* to count memory footprint */ + unsigned long long ext_mem[NR_EXTENT_CACHES]; + /* for read extent cache */ + unsigned long long hit_largest; + /* for block age extent cache */ + unsigned long long allocated_data_blocks; int ndirty_node, ndirty_dent, ndirty_meta, ndirty_imeta; int ndirty_data, ndirty_qdata; int inmem_pages; @@ -3838,10 +3864,10 @@ static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi) #define stat_other_skip_bggc_count(sbi) ((sbi)->other_skip_bggc++) #define stat_inc_dirty_inode(sbi, type) ((sbi)->ndirty_inode[type]++) #define stat_dec_dirty_inode(sbi, type) ((sbi)->ndirty_inode[type]--) -#define stat_inc_total_hit(sbi) (atomic64_inc(&(sbi)->total_hit_ext)) -#define stat_inc_rbtree_node_hit(sbi) (atomic64_inc(&(sbi)->read_hit_rbtree)) +#define stat_inc_total_hit(sbi, type) (atomic64_inc(&(sbi)->total_hit_ext[type])) +#define stat_inc_rbtree_node_hit(sbi, type) (atomic64_inc(&(sbi)->read_hit_rbtree[type])) #define stat_inc_largest_node_hit(sbi) (atomic64_inc(&(sbi)->read_hit_largest)) -#define stat_inc_cached_node_hit(sbi) (atomic64_inc(&(sbi)->read_hit_cached)) +#define stat_inc_cached_node_hit(sbi, type) (atomic64_inc(&(sbi)->read_hit_cached[type])) #define stat_inc_inline_xattr(inode) \ do { \ if (f2fs_has_inline_xattr(inode)) \ @@ -3967,10 +3993,10 @@ void f2fs_update_sit_info(struct f2fs_sb_info *sbi); #define stat_other_skip_bggc_count(sbi) do { } while (0) #define stat_inc_dirty_inode(sbi, type) do { } while (0) #define stat_dec_dirty_inode(sbi, type) do { } while (0) -#define stat_inc_total_hit(sbi) do { } while (0) -#define stat_inc_rbtree_node_hit(sbi) do { } while (0) +#define stat_inc_total_hit(sbi, type) do { } while (0) +#define stat_inc_rbtree_node_hit(sbi, type) do { } while (0) #define stat_inc_largest_node_hit(sbi) do { } while (0) -#define stat_inc_cached_node_hit(sbi) do { } while (0) +#define stat_inc_cached_node_hit(sbi, type) do { } while (0) #define stat_inc_inline_xattr(inode) do { } while (0) #define stat_dec_inline_xattr(inode) do { } while (0) #define stat_inc_inline_inode(inode) do { } while (0) @@ -4075,20 +4101,34 @@ struct rb_entry *f2fs_lookup_rb_tree_ret(struct rb_root_cached *root, bool force, bool *leftmost); bool f2fs_check_rb_tree_consistence(struct f2fs_sb_info *sbi, struct rb_root_cached *root, bool check_key); -unsigned int f2fs_shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink); -void f2fs_init_extent_tree(struct inode *inode, struct page *ipage); +void f2fs_init_extent_tree(struct inode *inode); void f2fs_drop_extent_tree(struct inode *inode); -unsigned int f2fs_destroy_extent_node(struct inode *inode); +void f2fs_destroy_extent_node(struct inode *inode); void f2fs_destroy_extent_tree(struct inode *inode); -bool f2fs_lookup_extent_cache(struct inode *inode, pgoff_t pgofs, - struct extent_info *ei); -void f2fs_update_extent_cache(struct dnode_of_data *dn); -void f2fs_update_extent_cache_range(struct dnode_of_data *dn, - pgoff_t fofs, block_t blkaddr, unsigned int len); void f2fs_init_extent_cache_info(struct f2fs_sb_info *sbi); int __init f2fs_create_extent_cache(void); void f2fs_destroy_extent_cache(void); +/* read extent cache ops */ +void f2fs_init_read_extent_tree(struct inode *inode, struct page *ipage); +bool f2fs_lookup_read_extent_cache(struct inode *inode, pgoff_t pgofs, + struct extent_info *ei); +void f2fs_update_read_extent_cache(struct dnode_of_data *dn); +void f2fs_update_read_extent_cache_range(struct dnode_of_data *dn, + pgoff_t fofs, block_t blkaddr, unsigned int len); +unsigned int f2fs_shrink_read_extent_tree(struct f2fs_sb_info *sbi, + int nr_shrink); + +/* block age extent cache ops */ +void f2fs_init_age_extent_tree(struct inode *inode); +bool f2fs_lookup_age_extent_cache(struct inode *inode, pgoff_t pgofs, + struct extent_info *ei); +void f2fs_update_age_extent_cache(struct dnode_of_data *dn); +void f2fs_update_age_extent_cache_range(struct dnode_of_data *dn, + pgoff_t fofs, unsigned int len); +unsigned int f2fs_shrink_age_extent_tree(struct f2fs_sb_info *sbi, + int nr_shrink); + /* * sysfs.c */ @@ -4152,12 +4192,16 @@ int f2fs_write_multi_pages(struct compress_ctx *cc, struct writeback_control *wbc, enum iostat_type io_type); int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index); +void f2fs_update_read_extent_tree_range_compressed(struct inode *inode, + pgoff_t fofs, block_t blkaddr, + unsigned int llen, unsigned int c_len); int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, unsigned nr_pages, sector_t *last_block_in_bio, bool is_readahead, bool for_write); struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc); void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed); void f2fs_put_page_dic(struct page *page); +unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn); int f2fs_init_compress_ctx(struct compress_ctx *cc); void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse); void f2fs_init_compress_info(struct f2fs_sb_info *sbi); @@ -4212,6 +4256,7 @@ static inline void f2fs_put_page_dic(struct page *page) { WARN_ON_ONCE(1); } +static inline unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn) { return 0; } static inline int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) { return 0; } static inline void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) { } static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; } @@ -4227,6 +4272,10 @@ static inline bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi, static inline void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino) { } #define inc_compr_inode_stat(inode) do { } while (0) +static inline void f2fs_update_read_extent_tree_range_compressed( + struct inode *inode, + pgoff_t fofs, block_t blkaddr, + unsigned int llen, unsigned int c_len) { } #endif static inline int set_compress_context(struct inode *inode) @@ -4296,26 +4345,6 @@ F2FS_FEATURE_FUNCS(casefold, CASEFOLD); F2FS_FEATURE_FUNCS(compression, COMPRESSION); F2FS_FEATURE_FUNCS(readonly, RO); -static inline bool f2fs_may_extent_tree(struct inode *inode) -{ - struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - - if (!test_opt(sbi, EXTENT_CACHE) || - is_inode_flag_set(inode, FI_NO_EXTENT) || - (is_inode_flag_set(inode, FI_COMPRESSED_FILE) && - !f2fs_sb_has_readonly(sbi))) - return false; - - /* - * for recovered files during mount do not create extents - * if shrinker is not registered. - */ - if (list_empty(&sbi->s_list)) - return false; - - return S_ISREG(inode->i_mode); -} - #ifdef CONFIG_BLK_DEV_ZONED static inline bool f2fs_blkz_is_seq(struct f2fs_sb_info *sbi, int devi, block_t blkaddr) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 36d8f0376f76..ef49b3684c0c 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -607,7 +607,8 @@ void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count) */ fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) + ofs; - f2fs_update_extent_cache_range(dn, fofs, 0, len); + f2fs_update_read_extent_cache_range(dn, fofs, 0, len); + f2fs_update_age_extent_cache_range(dn, fofs, nr_free); dec_valid_block_count(sbi, dn->inode, nr_free); } dn->ofs_in_node = ofs; @@ -1430,7 +1431,7 @@ static int f2fs_do_zero_range(struct dnode_of_data *dn, pgoff_t start, f2fs_set_data_blkaddr(dn); } - f2fs_update_extent_cache_range(dn, start, 0, index - start); + f2fs_update_read_extent_cache_range(dn, start, 0, index - start); return ret; } @@ -2590,7 +2591,7 @@ static int f2fs_defragment_range(struct f2fs_sb_info *sbi, struct f2fs_map_blocks map = { .m_next_extent = NULL, .m_seg_type = NO_CHECK_TYPE, .m_may_create = false }; - struct extent_info ei = {0, 0, 0}; + struct extent_info ei = {}; pgoff_t pg_start, pg_end, next_pgofs; unsigned int blk_per_seg = sbi->blocks_per_seg; unsigned int total = 0, sec_num; @@ -2622,7 +2623,7 @@ static int f2fs_defragment_range(struct f2fs_sb_info *sbi, * lookup mapping info in extent cache, skip defragmenting if physical * block addresses are continuous. */ - if (f2fs_lookup_extent_cache(inode, pg_start, &ei)) { + if (f2fs_lookup_read_extent_cache(inode, pg_start, &ei)) { if (ei.fofs + ei.len >= pg_end) goto out; } diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 85956d579c82..ab774c6fe035 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -1062,7 +1062,7 @@ static int ra_data_block(struct inode *inode, pgoff_t index) struct address_space *mapping = inode->i_mapping; struct dnode_of_data dn; struct page *page; - struct extent_info ei = {0, 0, 0}; + struct extent_info ei = {0, }; struct f2fs_io_info fio = { .sbi = sbi, .ino = inode->i_ino, @@ -1080,7 +1080,7 @@ static int ra_data_block(struct inode *inode, pgoff_t index) if (!page) return -ENOMEM; - if (f2fs_lookup_extent_cache(inode, index, &ei)) { + if (f2fs_lookup_read_extent_cache(inode, index, &ei)) { dn.data_blkaddr = ei.blk + index - ei.fofs; if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr, DATA_GENERIC_ENHANCE_READ))) { diff --git a/fs/f2fs/hash.c b/fs/f2fs/hash.c index e3beac546c63..2788ceeaf5c2 100644 --- a/fs/f2fs/hash.c +++ b/fs/f2fs/hash.c @@ -91,7 +91,7 @@ static u32 TEA_hash_name(const u8 *p, size_t len) /* * Compute @fname->hash. For all directories, @fname->disk_name must be set. * For casefolded directories, @fname->usr_fname must be set, and also - * @fname->cf_name if the filename is valid Unicode. + * @fname->cf_name if the filename is valid Unicode and is not "." or "..". */ void f2fs_hash_filename(const struct inode *dir, struct f2fs_filename *fname) { @@ -110,10 +110,11 @@ void f2fs_hash_filename(const struct inode *dir, struct f2fs_filename *fname) /* * If the casefolded name is provided, hash it instead of the * on-disk name. If the casefolded name is *not* provided, that - * should only be because the name wasn't valid Unicode, so fall - * back to treating the name as an opaque byte sequence. Note - * that to handle encrypted directories, the fallback must use - * usr_fname (plaintext) rather than disk_name (ciphertext). + * should only be because the name wasn't valid Unicode or was + * "." or "..", so fall back to treating the name as an opaque + * byte sequence. Note that to handle encrypted directories, + * the fallback must use usr_fname (plaintext) rather than + * disk_name (ciphertext). */ WARN_ON_ONCE(!fname->usr_fname->name); if (fname->cf_name.name) { diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index 29bf3e215f52..5bf4f1cccd71 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -260,8 +260,8 @@ static bool sanity_check_inode(struct inode *inode, struct page *node_page) return false; } - if (F2FS_I(inode)->extent_tree) { - struct extent_info *ei = &F2FS_I(inode)->extent_tree->largest; + if (fi->extent_tree[EX_READ]) { + struct extent_info *ei = &fi->extent_tree[EX_READ]->largest; if (ei->len && (!f2fs_is_valid_blkaddr(sbi, ei->blk, @@ -380,8 +380,6 @@ static int do_read_inode(struct inode *inode) fi->i_pino = le32_to_cpu(ri->i_pino); fi->i_dir_level = ri->i_dir_level; - f2fs_init_extent_tree(inode, node_page); - get_inline_info(inode, ri); fi->i_extra_isize = f2fs_has_extra_attr(inode) ? @@ -469,6 +467,11 @@ static int do_read_inode(struct inode *inode) F2FS_I(inode)->i_disk_time[1] = inode->i_ctime; F2FS_I(inode)->i_disk_time[2] = inode->i_mtime; F2FS_I(inode)->i_disk_time[3] = F2FS_I(inode)->i_crtime; + + /* Need all the flag bits */ + f2fs_init_read_extent_tree(inode, node_page); + f2fs_init_age_extent_tree(inode); + f2fs_put_page(node_page, 1); stat_inc_inline_xattr(inode); @@ -571,7 +574,7 @@ retry: void f2fs_update_inode(struct inode *inode, struct page *node_page) { struct f2fs_inode *ri; - struct extent_tree *et = F2FS_I(inode)->extent_tree; + struct extent_tree *et = F2FS_I(inode)->extent_tree[EX_READ]; f2fs_wait_on_page_writeback(node_page, NODE, true, true); set_page_dirty(node_page); @@ -590,7 +593,7 @@ void f2fs_update_inode(struct inode *inode, struct page *node_page) if (et) { read_lock(&et->lock); - set_raw_extent(&et->largest, &ri->i_ext); + set_raw_read_extent(&et->largest, &ri->i_ext); read_unlock(&et->lock); } else { memset(&ri->i_ext, 0, sizeof(ri->i_ext)); diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index fd4265c73b0d..14b2eb8a65e0 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -105,8 +105,6 @@ static struct inode *f2fs_new_inode(struct inode *dir, umode_t mode) } F2FS_I(inode)->i_inline_xattr_size = xattr_size; - f2fs_init_extent_tree(inode, NULL); - F2FS_I(inode)->i_flags = f2fs_mask_flags(mode, F2FS_I(dir)->i_flags & F2FS_FL_INHERITED); @@ -133,6 +131,8 @@ static struct inode *f2fs_new_inode(struct inode *dir, umode_t mode) f2fs_set_inode_flags(inode); + f2fs_init_extent_tree(inode); + trace_f2fs_new_inode(inode, 0); return inode; @@ -618,6 +618,8 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry) goto fail; } f2fs_delete_entry(de, page, dir, inode); + f2fs_unlock_op(sbi); + #ifdef CONFIG_UNICODE /* VFS negative dentries are incompatible with Encoding and * Case-insensitiveness. Eventually we'll want avoid @@ -628,8 +630,6 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry) if (IS_CASEFOLDED(dir)) d_invalidate(dentry); #endif - f2fs_unlock_op(sbi); - if (IS_DIRSYNC(dir)) f2fs_sync_fs(sbi->sb, 1); fail: diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 8145399e7c62..230c63c99b55 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -58,7 +58,7 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type) avail_ram = val.totalram - val.totalhigh; /* - * give 25%, 25%, 50%, 50%, 50% memory for each components respectively + * give 25%, 25%, 50%, 50%, 25%, 25% memory for each components respectively */ if (type == FREE_NIDS) { mem_size = (nm_i->nid_cnt[FREE_NID] * @@ -83,12 +83,16 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type) sizeof(struct ino_entry); mem_size >>= PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); - } else if (type == EXTENT_CACHE) { - mem_size = (atomic_read(&sbi->total_ext_tree) * + } else if (type == READ_EXTENT_CACHE || type == AGE_EXTENT_CACHE) { + enum extent_type etype = type == READ_EXTENT_CACHE ? + EX_READ : EX_BLOCK_AGE; + struct extent_tree_info *eti = &sbi->extent_tree[etype]; + + mem_size = (atomic_read(&eti->total_ext_tree) * sizeof(struct extent_tree) + - atomic_read(&sbi->total_ext_node) * + atomic_read(&eti->total_ext_node) * sizeof(struct extent_node)) >> PAGE_SHIFT; - res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); } else if (type == INMEM_PAGES) { /* it allows 20% / total_ram for inmemory pages */ mem_size = get_pages(sbi, F2FS_INMEM_PAGES); @@ -846,6 +850,26 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) dn->ofs_in_node = offset[level]; dn->node_page = npage[level]; dn->data_blkaddr = f2fs_data_blkaddr(dn); + + if (is_inode_flag_set(dn->inode, FI_COMPRESSED_FILE) && + f2fs_sb_has_readonly(sbi)) { + unsigned int c_len = f2fs_cluster_blocks_are_contiguous(dn); + block_t blkaddr; + + if (!c_len) + goto out; + + blkaddr = f2fs_data_blkaddr(dn); + if (blkaddr == COMPRESS_ADDR) + blkaddr = data_blkaddr(dn->inode, dn->node_page, + dn->ofs_in_node + 1); + + f2fs_update_read_extent_tree_range_compressed(dn->inode, + index, blkaddr, + F2FS_I(dn->inode)->i_cluster_size, + c_len); + } +out: return 0; release_pages: @@ -1339,8 +1363,8 @@ static int read_node_page(struct page *page, int op_flags) if (err) return err; - if (unlikely(ni.blk_addr == NULL_ADDR) || - is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN)) { + /* NEW_ADDR can be seen, after cp_error drops some dirty node pages */ + if (unlikely(ni.blk_addr == NULL_ADDR || ni.blk_addr == NEW_ADDR)) { ClearPageUptodate(page); return -ENOENT; } diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h index ff14a6e5ac1c..3de7891c8e79 100644 --- a/fs/f2fs/node.h +++ b/fs/f2fs/node.h @@ -148,7 +148,8 @@ enum mem_type { NAT_ENTRIES, /* indicates the cached nat entry */ DIRTY_DENTS, /* indicates dirty dentry pages */ INO_ENTRIES, /* indicates inode entries */ - EXTENT_CACHE, /* indicates extent cache */ + READ_EXTENT_CACHE, /* indicates read extent cache */ + AGE_EXTENT_CACHE, /* indicates age extent cache */ INMEM_PAGES, /* indicates inmemory pages */ DISCARD_CACHE, /* indicates memory of cached discard cmds */ COMPRESS_PAGE, /* indicates memory of cached compressed pages */ diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 109a0dd35a1b..175402ba5c04 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -536,8 +536,14 @@ void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg) return; /* try to shrink extent cache when there is no enough memory */ - if (!f2fs_available_free_memory(sbi, EXTENT_CACHE)) - f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER); + if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE)) + f2fs_shrink_read_extent_tree(sbi, + READ_EXTENT_CACHE_SHRINK_NUMBER); + + /* try to shrink age extent cache when there is no enough memory */ + if (!f2fs_available_free_memory(sbi, AGE_EXTENT_CACHE)) + f2fs_shrink_age_extent_tree(sbi, + AGE_EXTENT_CACHE_SHRINK_NUMBER); /* check the # of cached NAT entries */ if (!f2fs_available_free_memory(sbi, NAT_ENTRIES)) @@ -3292,10 +3298,28 @@ static int __get_segment_type_4(struct f2fs_io_info *fio) } } +static int __get_age_segment_type(struct inode *inode, pgoff_t pgofs) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_info ei = {}; + + if (f2fs_lookup_age_extent_cache(inode, pgofs, &ei)) { + if (!ei.age) + return NO_CHECK_TYPE; + if (ei.age <= sbi->hot_data_age_threshold) + return CURSEG_HOT_DATA; + if (ei.age <= sbi->warm_data_age_threshold) + return CURSEG_WARM_DATA; + return CURSEG_COLD_DATA; + } + return NO_CHECK_TYPE; +} + static int __get_segment_type_6(struct f2fs_io_info *fio) { if (fio->type == DATA) { struct inode *inode = fio->page->mapping->host; + int type; if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) return CURSEG_COLD_DATA_PINNED; @@ -3310,6 +3334,11 @@ static int __get_segment_type_6(struct f2fs_io_info *fio) } if (file_is_cold(inode) || f2fs_need_compress_data(inode)) return CURSEG_COLD_DATA; + + type = __get_age_segment_type(inode, fio->page->index); + if (type != NO_CHECK_TYPE) + return type; + if (file_is_hot(inode) || is_inode_flag_set(inode, FI_HOT_DATA) || f2fs_is_atomic_file(inode) || @@ -3421,6 +3450,9 @@ void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr)); + if (IS_DATASEG(type)) + atomic64_inc(&sbi->allocated_data_blocks); + up_write(&sit_i->sentry_lock); if (page && IS_NODESEG(type)) { @@ -3542,6 +3574,8 @@ void f2fs_outplace_write_data(struct dnode_of_data *dn, struct f2fs_summary sum; f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); + if (fio->io_type == FS_DATA_IO || fio->io_type == FS_CP_DATA_IO) + f2fs_update_age_extent_cache(dn); set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version); do_write_page(&sum, fio); f2fs_update_data_blkaddr(dn, fio->new_blkaddr); diff --git a/fs/f2fs/shrinker.c b/fs/f2fs/shrinker.c index dd3c3c7a90ec..83d6fb97dcae 100644 --- a/fs/f2fs/shrinker.c +++ b/fs/f2fs/shrinker.c @@ -28,10 +28,13 @@ static unsigned long __count_free_nids(struct f2fs_sb_info *sbi) return count > 0 ? count : 0; } -static unsigned long __count_extent_cache(struct f2fs_sb_info *sbi) +static unsigned long __count_extent_cache(struct f2fs_sb_info *sbi, + enum extent_type type) { - return atomic_read(&sbi->total_zombie_tree) + - atomic_read(&sbi->total_ext_node); + struct extent_tree_info *eti = &sbi->extent_tree[type]; + + return atomic_read(&eti->total_zombie_tree) + + atomic_read(&eti->total_ext_node); } unsigned long f2fs_shrink_count(struct shrinker *shrink, @@ -53,8 +56,11 @@ unsigned long f2fs_shrink_count(struct shrinker *shrink, } spin_unlock(&f2fs_list_lock); - /* count extent cache entries */ - count += __count_extent_cache(sbi); + /* count read extent cache entries */ + count += __count_extent_cache(sbi, EX_READ); + + /* count block age extent cache entries */ + count += __count_extent_cache(sbi, EX_BLOCK_AGE); /* count clean nat cache entries */ count += __count_nat_entries(sbi); @@ -100,7 +106,10 @@ unsigned long f2fs_shrink_scan(struct shrinker *shrink, sbi->shrinker_run_no = run_no; /* shrink extent cache entries */ - freed += f2fs_shrink_extent_tree(sbi, nr >> 1); + freed += f2fs_shrink_age_extent_tree(sbi, nr >> 2); + + /* shrink read extent cache entries */ + freed += f2fs_shrink_read_extent_tree(sbi, nr >> 2); /* shrink clean nat cache entries */ if (freed < nr) @@ -130,7 +139,9 @@ void f2fs_join_shrinker(struct f2fs_sb_info *sbi) void f2fs_leave_shrinker(struct f2fs_sb_info *sbi) { - f2fs_shrink_extent_tree(sbi, __count_extent_cache(sbi)); + f2fs_shrink_read_extent_tree(sbi, __count_extent_cache(sbi, EX_READ)); + f2fs_shrink_age_extent_tree(sbi, + __count_extent_cache(sbi, EX_BLOCK_AGE)); spin_lock(&f2fs_list_lock); list_del_init(&sbi->s_list); diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 89d90d366754..9650528d8f65 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -154,6 +154,7 @@ enum { Opt_atgc, Opt_gc_merge, Opt_nogc_merge, + Opt_age_extent_cache, Opt_err, }; @@ -229,6 +230,7 @@ static match_table_t f2fs_tokens = { {Opt_atgc, "atgc"}, {Opt_gc_merge, "gc_merge"}, {Opt_nogc_merge, "nogc_merge"}, + {Opt_age_extent_cache, "age_extent_cache"}, {Opt_err, NULL}, }; @@ -753,10 +755,10 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) set_opt(sbi, FASTBOOT); break; case Opt_extent_cache: - set_opt(sbi, EXTENT_CACHE); + set_opt(sbi, READ_EXTENT_CACHE); break; case Opt_noextent_cache: - clear_opt(sbi, EXTENT_CACHE); + clear_opt(sbi, READ_EXTENT_CACHE); break; case Opt_noinline_data: clear_opt(sbi, INLINE_DATA); @@ -1148,6 +1150,9 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) case Opt_nogc_merge: clear_opt(sbi, GC_MERGE); break; + case Opt_age_extent_cache: + set_opt(sbi, AGE_EXTENT_CACHE); + break; default: f2fs_err(sbi, "Unrecognized mount option \"%s\" or missing value", p); @@ -1817,10 +1822,12 @@ static int f2fs_show_options(struct seq_file *seq, struct dentry *root) seq_puts(seq, ",nobarrier"); if (test_opt(sbi, FASTBOOT)) seq_puts(seq, ",fastboot"); - if (test_opt(sbi, EXTENT_CACHE)) + if (test_opt(sbi, READ_EXTENT_CACHE)) seq_puts(seq, ",extent_cache"); else seq_puts(seq, ",noextent_cache"); + if (test_opt(sbi, AGE_EXTENT_CACHE)) + seq_puts(seq, ",age_extent_cache"); if (test_opt(sbi, DATA_FLUSH)) seq_puts(seq, ",data_flush"); @@ -1922,7 +1929,7 @@ static void default_options(struct f2fs_sb_info *sbi) set_opt(sbi, INLINE_XATTR); set_opt(sbi, INLINE_DATA); set_opt(sbi, INLINE_DENTRY); - set_opt(sbi, EXTENT_CACHE); + set_opt(sbi, READ_EXTENT_CACHE); set_opt(sbi, NOHEAP); clear_opt(sbi, DISABLE_CHECKPOINT); set_opt(sbi, MERGE_CHECKPOINT); @@ -2042,7 +2049,8 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) bool need_restart_gc = false, need_stop_gc = false; bool need_restart_ckpt = false, need_stop_ckpt = false; bool need_restart_flush = false, need_stop_flush = false; - bool no_extent_cache = !test_opt(sbi, EXTENT_CACHE); + bool no_read_extent_cache = !test_opt(sbi, READ_EXTENT_CACHE); + bool no_age_extent_cache = !test_opt(sbi, AGE_EXTENT_CACHE); bool disable_checkpoint = test_opt(sbi, DISABLE_CHECKPOINT); bool no_io_align = !F2FS_IO_ALIGNED(sbi); bool no_atgc = !test_opt(sbi, ATGC); @@ -2132,11 +2140,17 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) } /* disallow enable/disable extent_cache dynamically */ - if (no_extent_cache == !!test_opt(sbi, EXTENT_CACHE)) { + if (no_read_extent_cache == !!test_opt(sbi, READ_EXTENT_CACHE)) { err = -EINVAL; f2fs_warn(sbi, "switch extent_cache option is not allowed"); goto restore_opts; } + /* disallow enable/disable age extent_cache dynamically */ + if (no_age_extent_cache == !!test_opt(sbi, AGE_EXTENT_CACHE)) { + err = -EINVAL; + f2fs_warn(sbi, "switch age_extent_cache option is not allowed"); + goto restore_opts; + } if (no_io_align == !!F2FS_IO_ALIGNED(sbi)) { err = -EINVAL; diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 23eba7514c9c..c9217d7a941f 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -549,6 +549,24 @@ out: return count; } + if (!strcmp(a->attr.name, "hot_data_age_threshold")) { + if (t == 0 || t >= sbi->warm_data_age_threshold) + return -EINVAL; + if (t == *ui) + return count; + *ui = (unsigned int)t; + return count; + } + + if (!strcmp(a->attr.name, "warm_data_age_threshold")) { + if (t == 0 || t <= sbi->hot_data_age_threshold) + return -EINVAL; + if (t == *ui) + return count; + *ui = (unsigned int)t; + return count; + } + *ui = (unsigned int)t; return count; @@ -778,6 +796,10 @@ F2FS_RW_ATTR(ATGC_INFO, atgc_management, atgc_age_threshold, age_threshold); F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, gc_segment_mode, gc_segment_mode); F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, gc_reclaimed_segments, gc_reclaimed_segs); +/* For block age extent cache */ +F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, hot_data_age_threshold, hot_data_age_threshold); +F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, warm_data_age_threshold, warm_data_age_threshold); + #define ATTR_LIST(name) (&f2fs_attr_##name.attr) static struct attribute *f2fs_attrs[] = { ATTR_LIST(gc_urgent_sleep_time), @@ -853,6 +875,8 @@ static struct attribute *f2fs_attrs[] = { ATTR_LIST(atgc_age_threshold), ATTR_LIST(gc_segment_mode), ATTR_LIST(gc_reclaimed_segments), + ATTR_LIST(hot_data_age_threshold), + ATTR_LIST(warm_data_age_threshold), NULL, }; ATTRIBUTE_GROUPS(f2fs); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index e9a1543ba7cb..47eef2496230 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -208,10 +208,13 @@ static unsigned int fuse_req_hash(u64 unique) /** * A new request is available, wake fiq->waitq */ -static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq) +static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq, bool sync) __releases(fiq->lock) { - wake_up(&fiq->waitq); + if (sync) + wake_up_sync(&fiq->waitq); + else + wake_up(&fiq->waitq); kill_fasync(&fiq->fasync, SIGIO, POLL_IN); spin_unlock(&fiq->lock); } @@ -224,14 +227,14 @@ const struct fuse_iqueue_ops fuse_dev_fiq_ops = { EXPORT_SYMBOL_GPL(fuse_dev_fiq_ops); static void queue_request_and_unlock(struct fuse_iqueue *fiq, - struct fuse_req *req) + struct fuse_req *req, bool sync) __releases(fiq->lock) { req->in.h.len = sizeof(struct fuse_in_header) + fuse_len_args(req->args->in_numargs, (struct fuse_arg *) req->args->in_args); list_add_tail(&req->list, &fiq->pending); - fiq->ops->wake_pending_and_unlock(fiq); + fiq->ops->wake_pending_and_unlock(fiq, sync); } void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget, @@ -246,7 +249,7 @@ void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget, if (fiq->connected) { fiq->forget_list_tail->next = forget; fiq->forget_list_tail = forget; - fiq->ops->wake_forget_and_unlock(fiq); + fiq->ops->wake_forget_and_unlock(fiq, false); } else { kfree(forget); spin_unlock(&fiq->lock); @@ -266,7 +269,7 @@ static void flush_bg_queue(struct fuse_conn *fc) fc->active_background++; spin_lock(&fiq->lock); req->in.h.unique = fuse_get_unique(fiq); - queue_request_and_unlock(fiq, req); + queue_request_and_unlock(fiq, req, false); } } @@ -359,7 +362,7 @@ static int queue_interrupt(struct fuse_req *req) spin_unlock(&fiq->lock); return 0; } - fiq->ops->wake_interrupt_and_unlock(fiq); + fiq->ops->wake_interrupt_and_unlock(fiq, false); } else { spin_unlock(&fiq->lock); } @@ -426,7 +429,7 @@ static void __fuse_request_send(struct fuse_req *req) /* acquire extra reference, since request is still needed after fuse_request_end() */ __fuse_get_request(req); - queue_request_and_unlock(fiq, req); + queue_request_and_unlock(fiq, req, true); request_wait_answer(req); /* Pairs with smp_wmb() in fuse_request_end() */ @@ -601,7 +604,7 @@ static int fuse_simple_notify_reply(struct fuse_mount *fm, spin_lock(&fiq->lock); if (fiq->connected) { - queue_request_and_unlock(fiq, req); + queue_request_and_unlock(fiq, req, false); } else { err = -ENODEV; spin_unlock(&fiq->lock); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 69a631c12b15..e20c341864b5 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -412,19 +412,19 @@ struct fuse_iqueue_ops { /** * Signal that a forget has been queued */ - void (*wake_forget_and_unlock)(struct fuse_iqueue *fiq) + void (*wake_forget_and_unlock)(struct fuse_iqueue *fiq, bool sync) __releases(fiq->lock); /** * Signal that an INTERRUPT request has been queued */ - void (*wake_interrupt_and_unlock)(struct fuse_iqueue *fiq) + void (*wake_interrupt_and_unlock)(struct fuse_iqueue *fiq, bool sync) __releases(fiq->lock); /** * Signal that a request has been queued */ - void (*wake_pending_and_unlock)(struct fuse_iqueue *fiq) + void (*wake_pending_and_unlock)(struct fuse_iqueue *fiq, bool sync) __releases(fiq->lock); /** diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index b9cfb1165ff4..90a574ba92cf 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -971,7 +971,7 @@ static struct virtio_driver virtio_fs_driver = { #endif }; -static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq) +static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq, bool sync) __releases(fiq->lock) { struct fuse_forget_link *link; @@ -1006,7 +1006,8 @@ __releases(fiq->lock) kfree(link); } -static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq) +static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq, + bool sync) __releases(fiq->lock) { /* @@ -1221,7 +1222,8 @@ out: return ret; } -static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq) +static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq, + bool sync) __releases(fiq->lock) { unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */ diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c index 7183fe4b5f72..df2950500334 100644 --- a/fs/pstore/ram.c +++ b/fs/pstore/ram.c @@ -679,7 +679,7 @@ static int ramoops_parse_dt(struct platform_device *pdev, field = value; \ } - parse_u32("mem-type", pdata->record_size, pdata->mem_type); + parse_u32("mem-type", pdata->mem_type, pdata->mem_type); parse_u32("record-size", pdata->record_size, 0); parse_u32("console-size", pdata->console_size, 0); parse_u32("ftrace-size", pdata->ftrace_size, 0); diff --git a/include/crypto/hash.h b/include/crypto/hash.h index b2bc1e46e86a..14d75caa233d 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -149,7 +149,7 @@ struct ahash_alg { struct shash_desc { struct crypto_shash *tfm; - void *__ctx[] __aligned(ARCH_SLAB_MINALIGN); + void *__ctx[] __aligned(UL(16)); }; #define HASH_MAX_DIGESTSIZE 64 diff --git a/include/dt-bindings/leds/common.h b/include/dt-bindings/leds/common.h index 52b619d44ba2..3be89a7c20a9 100644 --- a/include/dt-bindings/leds/common.h +++ b/include/dt-bindings/leds/common.h @@ -60,6 +60,13 @@ #define LED_FUNCTION_MICMUTE "micmute" #define LED_FUNCTION_MUTE "mute" +/* Used for player LEDs as found on game controllers from e.g. Nintendo, Sony. */ +#define LED_FUNCTION_PLAYER1 "player-1" +#define LED_FUNCTION_PLAYER2 "player-2" +#define LED_FUNCTION_PLAYER3 "player-3" +#define LED_FUNCTION_PLAYER4 "player-4" +#define LED_FUNCTION_PLAYER5 "player-5" + /* Miscelleaus functions. Use functions above if you can. */ #define LED_FUNCTION_ACTIVITY "activity" #define LED_FUNCTION_ALARM "alarm" diff --git a/include/linux/debug_locks.h b/include/linux/debug_locks.h index edb5c186b0b7..ad06852cbd5d 100644 --- a/include/linux/debug_locks.h +++ b/include/linux/debug_locks.h @@ -5,6 +5,7 @@ #include #include #include +#include struct task_struct; diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index b9a8549f9a7f..c25005764d44 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -191,10 +191,10 @@ static inline int dma_declare_coherent_memory(struct device *dev, return -ENOSYS; } -#define dma_release_coherent_memory(dev) (0) #define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0) #define dma_release_from_dev_coherent(dev, order, vaddr) (0) #define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0) +static inline void dma_release_coherent_memory(struct device *dev) { } static inline void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size, dma_addr_t *dma_handle) diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index e203fdb020c4..a1b067f0c6ad 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -190,6 +190,8 @@ struct module; #ifdef BUILD_FIPS140_KO +#include + static inline int static_key_count(struct static_key *key) { return atomic_read(&key->enabled); diff --git a/include/linux/kasan-enabled.h b/include/linux/kasan-enabled.h new file mode 100644 index 000000000000..6f612d69ea0c --- /dev/null +++ b/include/linux/kasan-enabled.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_KASAN_ENABLED_H +#define _LINUX_KASAN_ENABLED_H + +#include + +#ifdef CONFIG_KASAN_HW_TAGS + +DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled); + +static __always_inline bool kasan_enabled(void) +{ + return static_branch_likely(&kasan_flag_enabled); +} + +static inline bool kasan_hw_tags_enabled(void) +{ + return kasan_enabled(); +} + +#else /* CONFIG_KASAN_HW_TAGS */ + +static inline bool kasan_enabled(void) +{ + return IS_ENABLED(CONFIG_KASAN); +} + +static inline bool kasan_hw_tags_enabled(void) +{ + return false; +} + +#endif /* CONFIG_KASAN_HW_TAGS */ + +#endif /* LINUX_KASAN_ENABLED_H */ diff --git a/include/linux/kasan.h b/include/linux/kasan.h index b59675cc19b7..99dc14b5ad44 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -3,6 +3,8 @@ #define _LINUX_KASAN_H #include +#include +#include #include #include @@ -82,33 +84,11 @@ static inline void kasan_disable_current(void) {} #ifdef CONFIG_KASAN_HW_TAGS -DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled); - -static __always_inline bool kasan_enabled(void) -{ - return static_branch_likely(&kasan_flag_enabled); -} - -static inline bool kasan_hw_tags_enabled(void) -{ - return kasan_enabled(); -} - void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags); void kasan_free_pages(struct page *page, unsigned int order); #else /* CONFIG_KASAN_HW_TAGS */ -static inline bool kasan_enabled(void) -{ - return IS_ENABLED(CONFIG_KASAN); -} - -static inline bool kasan_hw_tags_enabled(void) -{ - return false; -} - static __always_inline void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags) { diff --git a/include/linux/mm.h b/include/linux/mm.h index 0623428c2313..dfefcfa1d6a4 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1760,6 +1760,12 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, #ifdef CONFIG_SPECULATIVE_PAGE_FAULT static inline void vm_write_begin(struct vm_area_struct *vma) { + /* + * Isolated vma might be freed without exclusive mmap_lock but + * speculative page fault handler still needs to know it was changed. + */ + if (!RB_EMPTY_NODE(&vma->vm_rb)) + mmap_assert_write_locked(vma->vm_mm); /* * The reads never spins and preemption * disablement is not required. diff --git a/include/linux/module.h b/include/linux/module.h index c5c3ce08f646..4cd6d889d5ba 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -293,8 +293,7 @@ extern typeof(name) __mod_##type##__##name##_device_table \ * files require multiple MODULE_FIRMWARE() specifiers */ #define MODULE_FIRMWARE(_firmware) MODULE_INFO(firmware, _firmware) -#define _MODULE_IMPORT_NS(ns) MODULE_INFO(import_ns, #ns) -#define MODULE_IMPORT_NS(ns) _MODULE_IMPORT_NS(ns) +#define MODULE_IMPORT_NS(ns) MODULE_INFO(import_ns, __stringify(ns)) struct notifier_block; diff --git a/include/linux/printk.h b/include/linux/printk.h index f589b8b60806..14d13ecaa8f8 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -6,7 +6,6 @@ #include #include #include -#include #include extern const char linux_banner[]; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 59a6bf66421f..f73efb3057e6 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1432,6 +1432,11 @@ static inline unsigned int skb_end_offset(const struct sk_buff *skb) { return skb->end; } + +static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset) +{ + skb->end = offset; +} #else static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) { @@ -1442,6 +1447,11 @@ static inline unsigned int skb_end_offset(const struct sk_buff *skb) { return skb->end - skb->head; } + +static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset) +{ + skb->end = skb->head + offset; +} #endif /* Internal */ @@ -1661,19 +1671,19 @@ static inline int skb_unclone(struct sk_buff *skb, gfp_t pri) return 0; } -/* This variant of skb_unclone() makes sure skb->truesize is not changed */ +/* This variant of skb_unclone() makes sure skb->truesize + * and skb_end_offset() are not changed, whenever a new skb->head is needed. + * + * Indeed there is no guarantee that ksize(kmalloc(X)) == ksize(kmalloc(X)) + * when various debugging features are in place. + */ +int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri); static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) { might_sleep_if(gfpflags_allow_blocking(pri)); - if (skb_cloned(skb)) { - unsigned int save = skb->truesize; - int res; - - res = pskb_expand_head(skb, 0, 0, pri); - skb->truesize = save; - return res; - } + if (skb_cloned(skb)) + return __skb_unclone_keeptruesize(skb, pri); return 0; } diff --git a/include/linux/slab.h b/include/linux/slab.h index dd6897f62010..36217130c3ae 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -217,6 +217,18 @@ static inline void __check_heap_object(const void *ptr, unsigned long n, #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) #endif +/* + * Arches can define this function if they want to decide the minimum slab + * alignment at runtime. The value returned by the function must be a power + * of two and >= ARCH_SLAB_MINALIGN. + */ +#ifndef arch_slab_minalign +static inline unsigned int arch_slab_minalign(void) +{ + return ARCH_SLAB_MINALIGN; +} +#endif + /* * kmalloc and friends return ARCH_KMALLOC_MINALIGN aligned * pointers. kmem_cache_alloc and friends return ARCH_SLAB_MINALIGN diff --git a/include/linux/wait.h b/include/linux/wait.h index 1663e47681a3..e9966f3929f6 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -219,6 +219,7 @@ void __wake_up_pollfree(struct wait_queue_head *wq_head); #define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL) #define wake_up_interruptible_all(x) __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL) #define wake_up_interruptible_sync(x) __wake_up_sync((x), TASK_INTERRUPTIBLE) +#define wake_up_sync(x) __wake_up_sync((x), TASK_NORMAL) /* * Wakeup macros to be used to report events to the targets. diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index df293bc7f03b..e927889c7fd5 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h @@ -52,6 +52,8 @@ TRACE_DEFINE_ENUM(CP_DISCARD); TRACE_DEFINE_ENUM(CP_TRIMMED); TRACE_DEFINE_ENUM(CP_PAUSE); TRACE_DEFINE_ENUM(CP_RESIZE); +TRACE_DEFINE_ENUM(EX_READ); +TRACE_DEFINE_ENUM(EX_BLOCK_AGE); #define show_block_type(type) \ __print_symbolic(type, \ @@ -162,6 +164,11 @@ TRACE_DEFINE_ENUM(CP_RESIZE); { COMPRESS_ZSTD, "ZSTD" }, \ { COMPRESS_LZORLE, "LZO-RLE" }) +#define show_extent_type(type) \ + __print_symbolic(type, \ + { EX_READ, "Read" }, \ + { EX_BLOCK_AGE, "Block Age" }) + struct f2fs_sb_info; struct f2fs_io_info; struct extent_info; @@ -1526,28 +1533,31 @@ TRACE_EVENT(f2fs_issue_flush, TRACE_EVENT(f2fs_lookup_extent_tree_start, - TP_PROTO(struct inode *inode, unsigned int pgofs), + TP_PROTO(struct inode *inode, unsigned int pgofs, enum extent_type type), - TP_ARGS(inode, pgofs), + TP_ARGS(inode, pgofs, type), TP_STRUCT__entry( __field(dev_t, dev) __field(ino_t, ino) __field(unsigned int, pgofs) + __field(enum extent_type, type) ), TP_fast_assign( __entry->dev = inode->i_sb->s_dev; __entry->ino = inode->i_ino; __entry->pgofs = pgofs; + __entry->type = type; ), - TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u", + TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, type = %s", show_dev_ino(__entry), - __entry->pgofs) + __entry->pgofs, + show_extent_type(__entry->type)) ); -TRACE_EVENT_CONDITION(f2fs_lookup_extent_tree_end, +TRACE_EVENT_CONDITION(f2fs_lookup_read_extent_tree_end, TP_PROTO(struct inode *inode, unsigned int pgofs, struct extent_info *ei), @@ -1561,8 +1571,8 @@ TRACE_EVENT_CONDITION(f2fs_lookup_extent_tree_end, __field(ino_t, ino) __field(unsigned int, pgofs) __field(unsigned int, fofs) - __field(u32, blk) __field(unsigned int, len) + __field(u32, blk) ), TP_fast_assign( @@ -1570,25 +1580,65 @@ TRACE_EVENT_CONDITION(f2fs_lookup_extent_tree_end, __entry->ino = inode->i_ino; __entry->pgofs = pgofs; __entry->fofs = ei->fofs; - __entry->blk = ei->blk; __entry->len = ei->len; + __entry->blk = ei->blk; ), TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, " - "ext_info(fofs: %u, blk: %u, len: %u)", + "read_ext_info(fofs: %u, len: %u, blk: %u)", show_dev_ino(__entry), __entry->pgofs, __entry->fofs, - __entry->blk, - __entry->len) + __entry->len, + __entry->blk) ); -TRACE_EVENT(f2fs_update_extent_tree_range, +TRACE_EVENT_CONDITION(f2fs_lookup_age_extent_tree_end, - TP_PROTO(struct inode *inode, unsigned int pgofs, block_t blkaddr, - unsigned int len), + TP_PROTO(struct inode *inode, unsigned int pgofs, + struct extent_info *ei), - TP_ARGS(inode, pgofs, blkaddr, len), + TP_ARGS(inode, pgofs, ei), + + TP_CONDITION(ei), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(unsigned int, pgofs) + __field(unsigned int, fofs) + __field(unsigned int, len) + __field(unsigned long long, age) + __field(unsigned long long, blocks) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->pgofs = pgofs; + __entry->fofs = ei->fofs; + __entry->len = ei->len; + __entry->age = ei->age; + __entry->blocks = ei->last_blocks; + ), + + TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, " + "age_ext_info(fofs: %u, len: %u, age: %llu, blocks: %llu)", + show_dev_ino(__entry), + __entry->pgofs, + __entry->fofs, + __entry->len, + __entry->age, + __entry->blocks) +); + +TRACE_EVENT(f2fs_update_read_extent_tree_range, + + TP_PROTO(struct inode *inode, unsigned int pgofs, unsigned int len, + block_t blkaddr, + unsigned int c_len), + + TP_ARGS(inode, pgofs, len, blkaddr, c_len), TP_STRUCT__entry( __field(dev_t, dev) @@ -1596,70 +1646,115 @@ TRACE_EVENT(f2fs_update_extent_tree_range, __field(unsigned int, pgofs) __field(u32, blk) __field(unsigned int, len) + __field(unsigned int, c_len) ), TP_fast_assign( __entry->dev = inode->i_sb->s_dev; __entry->ino = inode->i_ino; __entry->pgofs = pgofs; - __entry->blk = blkaddr; __entry->len = len; + __entry->blk = blkaddr; + __entry->c_len = c_len; ), TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, " - "blkaddr = %u, len = %u", + "len = %u, blkaddr = %u, c_len = %u", show_dev_ino(__entry), __entry->pgofs, + __entry->len, __entry->blk, - __entry->len) + __entry->c_len) +); + +TRACE_EVENT(f2fs_update_age_extent_tree_range, + + TP_PROTO(struct inode *inode, unsigned int pgofs, unsigned int len, + unsigned long long age, + unsigned long long last_blks), + + TP_ARGS(inode, pgofs, len, age, last_blks), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(unsigned int, pgofs) + __field(unsigned int, len) + __field(unsigned long long, age) + __field(unsigned long long, blocks) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->pgofs = pgofs; + __entry->len = len; + __entry->age = age; + __entry->blocks = last_blks; + ), + + TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, " + "len = %u, age = %llu, blocks = %llu", + show_dev_ino(__entry), + __entry->pgofs, + __entry->len, + __entry->age, + __entry->blocks) ); TRACE_EVENT(f2fs_shrink_extent_tree, TP_PROTO(struct f2fs_sb_info *sbi, unsigned int node_cnt, - unsigned int tree_cnt), + unsigned int tree_cnt, enum extent_type type), - TP_ARGS(sbi, node_cnt, tree_cnt), + TP_ARGS(sbi, node_cnt, tree_cnt, type), TP_STRUCT__entry( __field(dev_t, dev) __field(unsigned int, node_cnt) __field(unsigned int, tree_cnt) + __field(enum extent_type, type) ), TP_fast_assign( __entry->dev = sbi->sb->s_dev; __entry->node_cnt = node_cnt; __entry->tree_cnt = tree_cnt; + __entry->type = type; ), - TP_printk("dev = (%d,%d), shrunk: node_cnt = %u, tree_cnt = %u", + TP_printk("dev = (%d,%d), shrunk: node_cnt = %u, tree_cnt = %u, type = %s", show_dev(__entry->dev), __entry->node_cnt, - __entry->tree_cnt) + __entry->tree_cnt, + show_extent_type(__entry->type)) ); TRACE_EVENT(f2fs_destroy_extent_tree, - TP_PROTO(struct inode *inode, unsigned int node_cnt), + TP_PROTO(struct inode *inode, unsigned int node_cnt, + enum extent_type type), - TP_ARGS(inode, node_cnt), + TP_ARGS(inode, node_cnt, type), TP_STRUCT__entry( __field(dev_t, dev) __field(ino_t, ino) __field(unsigned int, node_cnt) + __field(enum extent_type, type) ), TP_fast_assign( __entry->dev = inode->i_sb->s_dev; __entry->ino = inode->i_ino; __entry->node_cnt = node_cnt; + __entry->type = type; ), - TP_printk("dev = (%d,%d), ino = %lu, destroyed: node_cnt = %u", + TP_printk("dev = (%d,%d), ino = %lu, destroyed: node_cnt = %u, type = %s", show_dev_ino(__entry), - __entry->node_cnt) + __entry->node_cnt, + show_extent_type(__entry->type)) ); DECLARE_EVENT_CLASS(f2fs_sync_dirty_inodes, diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index d428f0137c49..a26dbefdf294 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -48,7 +48,9 @@ {(unsigned long)__GFP_WRITE, "__GFP_WRITE"}, \ {(unsigned long)__GFP_RECLAIM, "__GFP_RECLAIM"}, \ {(unsigned long)__GFP_DIRECT_RECLAIM, "__GFP_DIRECT_RECLAIM"},\ - {(unsigned long)__GFP_KSWAPD_RECLAIM, "__GFP_KSWAPD_RECLAIM"}\ + {(unsigned long)__GFP_KSWAPD_RECLAIM, "__GFP_KSWAPD_RECLAIM"},\ + {(unsigned long)__GFP_ZEROTAGS, "__GFP_ZEROTAGS"}, \ + {(unsigned long)__GFP_SKIP_KASAN_POISON,"__GFP_SKIP_KASAN_POISON"}\ #define show_gfp_flags(flags) \ (flags) ? __print_flags(flags, "|", \ diff --git a/include/trace/hooks/dmabuf.h b/include/trace/hooks/dmabuf.h new file mode 100644 index 000000000000..0182960efca2 --- /dev/null +++ b/include/trace/hooks/dmabuf.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM dmabuf + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH trace/hooks + +#if !defined(_TRACE_HOOK_DMABUF_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_DMABUF_H + +#include + +struct dma_buf_sysfs_entry; +DECLARE_RESTRICTED_HOOK(android_rvh_dma_buf_stats_teardown, + TP_PROTO(struct dma_buf_sysfs_entry *sysfs_entry, bool *skip_sysfs_release), + TP_ARGS(sysfs_entry, skip_sysfs_release), 1); +#endif /* _TRACE_HOOK_DMABUF_H */ +/* This part must be outside protection */ +#include + diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index 3f32c876441f..a4b855e0a81a 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -287,6 +287,14 @@ DECLARE_HOOK(android_vh_set_shmem_page_flag, DECLARE_HOOK(android_vh_remove_vmalloc_stack, TP_PROTO(struct vm_struct *vm), TP_ARGS(vm)); +DECLARE_HOOK(android_vh_alloc_pages_reclaim_bypass, + TP_PROTO(gfp_t gfp_mask, int order, int alloc_flags, + int migratetype, struct page **page), + TP_ARGS(gfp_mask, order, alloc_flags, migratetype, page)); +DECLARE_HOOK(android_vh_alloc_pages_failure_bypass, + TP_PROTO(gfp_t gfp_mask, int order, int alloc_flags, + int migratetype, struct page **page), + TP_ARGS(gfp_mask, order, alloc_flags, migratetype, page)); DECLARE_HOOK(android_vh_test_clear_look_around_ref, TP_PROTO(struct page *page), TP_ARGS(page)); diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h index 6488dee32a88..5f63128bb5c6 100644 --- a/include/trace/hooks/sched.h +++ b/include/trace/hooks/sched.h @@ -391,6 +391,10 @@ DECLARE_HOOK(android_vh_setscheduler_uclamp, TP_PROTO(struct task_struct *tsk, int clamp_id, unsigned int value), TP_ARGS(tsk, clamp_id, value)); +DECLARE_HOOK(android_vh_mmput, + TP_PROTO(void *unused), + TP_ARGS(unused)); + DECLARE_HOOK(android_vh_sched_pelt_multiplier, TP_PROTO(unsigned int old, unsigned int cur, int *ret), TP_ARGS(old, cur, ret)); diff --git a/init/Kconfig b/init/Kconfig index 47be071808e8..976ff1d27c5e 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -131,6 +131,20 @@ config COMPILE_TEST here. If you are a user/distributor, say N here to exclude useless drivers to be distributed. +config WERROR + bool "Compile the kernel with warnings as errors" + default y + help + A kernel build should not cause any compiler warnings, and this + enables the '-Werror' flag to enforce that rule by default. + + However, if you have a new (or very old) compiler with odd and + unusual warnings, or you have some architecture with problems, + you may need to disable this config option in order to + successfully build the kernel. + + If in doubt, say Y. + config UAPI_HEADER_TEST bool "Compile test UAPI headers" depends on HEADERS_INSTALL && CC_CAN_LINK diff --git a/kernel/bpf/bpf_lru_list.h b/kernel/bpf/bpf_lru_list.h index 6b12f06ee18c..4ea227c9c1ad 100644 --- a/kernel/bpf/bpf_lru_list.h +++ b/kernel/bpf/bpf_lru_list.h @@ -4,6 +4,7 @@ #ifndef __BPF_LRU_LIST_H_ #define __BPF_LRU_LIST_H_ +#include #include #include diff --git a/kernel/fork.c b/kernel/fork.c index b2ab89589a7d..d515aa5b7eb5 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1150,8 +1150,10 @@ void mmput(struct mm_struct *mm) { might_sleep(); - if (atomic_dec_and_test(&mm->mm_users)) + if (atomic_dec_and_test(&mm->mm_users)) { + trace_android_vh_mmput(NULL); __mmput(mm); + } } EXPORT_SYMBOL_GPL(mmput); diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index a5ba06a86358..4e999807b6c7 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -59,6 +59,7 @@ choice config KASAN_GENERIC bool "Generic mode" depends on HAVE_ARCH_KASAN && CC_HAS_KASAN_GENERIC + depends on CC_HAS_WORKING_NOSANITIZE_ADDRESS select SLUB_DEBUG if SLUB select CONSTRUCTORS help @@ -79,6 +80,7 @@ config KASAN_GENERIC config KASAN_SW_TAGS bool "Software tag-based mode" depends on HAVE_ARCH_KASAN_SW_TAGS && CC_HAS_KASAN_SW_TAGS + depends on CC_HAS_WORKING_NOSANITIZE_ADDRESS select SLUB_DEBUG if SLUB select CONSTRUCTORS help diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 93293eb04df9..5a1bf311b913 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -53,6 +53,10 @@ #include #include "kstrtox.h" +/* Disable pointer hashing if requested */ +bool no_hash_pointers __ro_after_init; +EXPORT_SYMBOL_GPL(no_hash_pointers); + static unsigned long long simple_strntoull(const char *startp, size_t max_chars, char **endp, unsigned int base) { @@ -849,6 +853,19 @@ static char *ptr_to_id(char *buf, char *end, const void *ptr, return pointer_string(buf, end, (const void *)hashval, spec); } +static char *default_pointer(char *buf, char *end, const void *ptr, + struct printf_spec spec) +{ + /* + * default is to _not_ leak addresses, so hash before printing, + * unless no_hash_pointers is specified on the command line. + */ + if (unlikely(no_hash_pointers)) + return pointer_string(buf, end, ptr, spec); + + return ptr_to_id(buf, end, ptr, spec); +} + int kptr_restrict __read_mostly; static noinline_for_stack @@ -858,7 +875,7 @@ char *restricted_pointer(char *buf, char *end, const void *ptr, switch (kptr_restrict) { case 0: /* Handle as %p, hash and do _not_ leak addresses. */ - return ptr_to_id(buf, end, ptr, spec); + return default_pointer(buf, end, ptr, spec); case 1: { const struct cred *cred; @@ -2118,10 +2135,6 @@ char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode, return widen_string(buf, buf - buf_start, end, spec); } -/* Disable pointer hashing if requested */ -bool no_hash_pointers __ro_after_init; -EXPORT_SYMBOL_GPL(no_hash_pointers); - static int __init no_hash_pointers_enable(char *str) { no_hash_pointers = true; @@ -2339,7 +2352,7 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, case 'e': /* %pe with a non-ERR_PTR gets treated as plain %p */ if (!IS_ERR(ptr)) - break; + return default_pointer(buf, end, ptr, spec); return err_ptr(buf, end, ptr, spec); case 'u': case 'k': @@ -2349,16 +2362,9 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, default: return error_string(buf, end, "(einval)", spec); } + default: + return default_pointer(buf, end, ptr, spec); } - - /* - * default is to _not_ leak addresses, so hash before printing, - * unless no_hash_pointers is specified on the command line. - */ - if (unlikely(no_hash_pointers)) - return pointer_string(buf, end, ptr, spec); - else - return ptr_to_id(buf, end, ptr, spec); } /* diff --git a/mm/Kconfig b/mm/Kconfig index 6a020a0d7bc5..03bfe7bd8183 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -843,7 +843,7 @@ config SPECULATIVE_PAGE_FAULT bool "Speculative page faults" default y depends on ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT - depends on MMU && SMP + depends on MMU && SMP && !NUMA help Try to handle user space page faults without holding the mmap_sem. diff --git a/mm/cma.c b/mm/cma.c index a7638d7487c4..44ca2b65b427 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -47,6 +47,7 @@ extern void lru_cache_enable(void); struct cma cma_areas[MAX_CMA_AREAS]; unsigned cma_area_count; +static DEFINE_MUTEX(cma_mutex); phys_addr_t cma_get_base(const struct cma *cma) { @@ -511,7 +512,9 @@ struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, mutex_unlock(&cma->lock); pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit); + mutex_lock(&cma_mutex); ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA, gfp_mask, &info); + mutex_unlock(&cma_mutex); cma_info.nr_migrated += info.nr_migrated; cma_info.nr_reclaimed += info.nr_reclaimed; cma_info.nr_mapped += info.nr_mapped; diff --git a/mm/damon/core.c b/mm/damon/core.c index 1dd153c31c9e..b194846f809b 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -155,6 +155,7 @@ struct damon_target *damon_new_target(unsigned long id) t->id = id; t->nr_regions = 0; INIT_LIST_HEAD(&t->regions_list); + INIT_LIST_HEAD(&t->list); return t; } diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 5b899601e56c..f2772fc740c0 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -721,6 +721,9 @@ static int dbgfs_mk_context(char *name) return -ENOENT; new_dir = debugfs_create_dir(name, root); + /* Below check is required for a potential duplicated name case */ + if (IS_ERR(new_dir)) + return PTR_ERR(new_dir); dbgfs_dirs[dbgfs_nr_ctxs] = new_dir; new_ctx = dbgfs_new_ctx(); @@ -784,6 +787,7 @@ static int dbgfs_rm_context(char *name) struct dentry *root, *dir, **new_dirs; struct damon_ctx **new_ctxs; int i, j; + int ret = 0; if (damon_nr_running_ctxs()) return -EBUSY; @@ -798,14 +802,16 @@ static int dbgfs_rm_context(char *name) new_dirs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_dirs), GFP_KERNEL); - if (!new_dirs) - return -ENOMEM; + if (!new_dirs) { + ret = -ENOMEM; + goto out_dput; + } new_ctxs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_ctxs), GFP_KERNEL); if (!new_ctxs) { - kfree(new_dirs); - return -ENOMEM; + ret = -ENOMEM; + goto out_new_dirs; } for (i = 0, j = 0; i < dbgfs_nr_ctxs; i++) { @@ -825,7 +831,13 @@ static int dbgfs_rm_context(char *name) dbgfs_ctxs = new_ctxs; dbgfs_nr_ctxs--; - return 0; + goto out_dput; + +out_new_dirs: + kfree(new_dirs); +out_dput: + dput(dir); + return ret; } static ssize_t dbgfs_rm_context_write(struct file *file, diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 89b6468da2b9..c0dec53b2330 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -375,6 +375,11 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, if (pmd_huge(*pmd)) { ptl = pmd_lock(walk->mm, pmd); + if (!pmd_present(*pmd)) { + spin_unlock(ptl); + return 0; + } + if (pmd_huge(*pmd)) { damon_pmdp_mkold(pmd, walk->mm, addr); spin_unlock(ptl); @@ -410,8 +415,7 @@ static void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, if (pte_young(entry)) { referenced = true; entry = pte_mkold(entry); - huge_ptep_set_access_flags(vma, addr, pte, entry, - vma->vm_flags & VM_WRITE); + set_huge_pte_at(mm, addr, pte, entry); } #ifdef CONFIG_MMU_NOTIFIER @@ -506,6 +510,11 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, #ifdef CONFIG_TRANSPARENT_HUGEPAGE if (pmd_huge(*pmd)) { ptl = pmd_lock(walk->mm, pmd); + if (!pmd_present(*pmd)) { + spin_unlock(ptl); + return 0; + } + if (!pmd_huge(*pmd)) { spin_unlock(ptl); goto regular_page; diff --git a/mm/filemap.c b/mm/filemap.c index 41bff800f5bb..aeb45f36aba1 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2706,7 +2706,9 @@ static struct file *do_async_mmap_readahead(struct vm_fault *vmf, * it in the page cache, and handles the special cases reasonably without * having a lot of duplicated code. * - * vma->vm_mm->mmap_lock must be held on entry. + * If FAULT_FLAG_SPECULATIVE is set, this function runs with elevated vma + * refcount and with mmap lock not held. + * Otherwise, vma->vm_mm->mmap_lock must be held on entry. * * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock * may be dropped before doing I/O or by lock_page_maybe_drop_mmap(). @@ -2732,6 +2734,47 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) vm_fault_t ret = 0; bool retry = false; + if (vmf->flags & FAULT_FLAG_SPECULATIVE) { + page = find_get_page(mapping, offset); + if (unlikely(!page) || unlikely(PageReadahead(page))) + return VM_FAULT_RETRY; + + if (!trylock_page(page)) + return VM_FAULT_RETRY; + + if (unlikely(compound_head(page)->mapping != mapping)) + goto page_unlock; + VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page); + if (unlikely(!PageUptodate(page))) + goto page_unlock; + + max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); + if (unlikely(offset >= max_off)) + goto page_unlock; + + /* + * Update readahead mmap_miss statistic. + * + * Note that we are not sure if finish_fault() will + * manage to complete the transaction. If it fails, + * we'll come back to filemap_fault() non-speculative + * case which will update mmap_miss a second time. + * This is not ideal, we would prefer to guarantee the + * update will happen exactly once. + */ + if (!(vmf->vma->vm_flags & VM_RAND_READ) && ra->ra_pages) { + unsigned int mmap_miss = READ_ONCE(ra->mmap_miss); + if (mmap_miss) + WRITE_ONCE(ra->mmap_miss, --mmap_miss); + } + + vmf->page = page; + return VM_FAULT_LOCKED; +page_unlock: + unlock_page(page); + return VM_FAULT_RETRY; + } + max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); if (unlikely(offset >= max_off)) return VM_FAULT_SIGBUS; @@ -2874,11 +2917,6 @@ static bool filemap_map_pmd(struct vm_fault *vmf, struct page *page) } if (pmd_none(*vmf->pmd)) { - if (vmf->flags & FAULT_FLAG_SPECULATIVE) { - unlock_page(page); - put_page(page); - return true; - } vmf->ptl = pmd_lock(mm, vmf->pmd); if (likely(pmd_none(*vmf->pmd))) { mm_inc_nr_ptes(mm); @@ -2976,20 +3014,16 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, XA_STATE(xas, &mapping->i_pages, start_pgoff); struct page *head, *page; unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss); - vm_fault_t ret = 0; + vm_fault_t ret = (vmf->flags & FAULT_FLAG_SPECULATIVE) ? + VM_FAULT_RETRY : 0; rcu_read_lock(); head = first_map_page(mapping, &xas, end_pgoff); if (!head) goto out; - if (filemap_map_pmd(vmf, head)) { - if (pmd_none(*vmf->pmd) && - vmf->flags & FAULT_FLAG_SPECULATIVE) { - ret = VM_FAULT_RETRY; - goto out; - } - + if (!(vmf->flags & FAULT_FLAG_SPECULATIVE) && + filemap_map_pmd(vmf, head)) { ret = VM_FAULT_NOPAGE; goto out; } @@ -2998,7 +3032,6 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, if (!pte_map_lock_addr(vmf, addr)) { unlock_page(head); put_page(head); - ret = VM_FAULT_RETRY; goto out; } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index cece54f4fb4a..035ef50be130 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5779,6 +5779,7 @@ void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) if (start >= end) return; + flush_cache_range(vma, start, end); /* * No need to call adjust_range_if_pmd_sharing_possible(), because * we have already done the PUD_SIZE alignment. diff --git a/mm/khugepaged.c b/mm/khugepaged.c index edecf6b20eb9..6ddc2761a408 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1484,6 +1484,8 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) if (!pmd) goto drop_hpage; + vm_write_begin(vma); + /* * We need to lock the mapping so that from here on, only GUP-fast and * hardware page walks can access the parts of the page tables that @@ -1547,6 +1549,7 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) haddr + HPAGE_PMD_SIZE); mmu_notifier_invalidate_range_start(&range); _pmd = pmdp_collapse_flush(vma, haddr, pmd); + vm_write_end(vma); mm_dec_nr_ptes(mm); tlb_remove_table_sync_one(); mmu_notifier_invalidate_range_end(&range); @@ -1561,6 +1564,7 @@ drop_hpage: abort: pte_unmap_unlock(start_pte, ptl); + vm_write_end(vma); i_mmap_unlock_write(vma->vm_file->f_mapping); goto drop_hpage; } @@ -1636,6 +1640,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) if (!khugepaged_test_exit(mm)) { struct mmu_notifier_range range; + vm_write_begin(vma); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm, addr, @@ -1643,6 +1648,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) mmu_notifier_invalidate_range_start(&range); /* assume page table is clear */ _pmd = pmdp_collapse_flush(vma, addr, pmd); + vm_write_end(vma); mm_dec_nr_ptes(mm); tlb_remove_table_sync_one(); pte_free(mm, pmd_pgtable(_pmd)); diff --git a/mm/madvise.c b/mm/madvise.c index cfa79899bb7c..b3761ca637bb 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -38,6 +38,7 @@ struct madvise_walk_private { struct mmu_gather *tlb; bool pageout; + bool can_pageout_file; }; /* @@ -313,6 +314,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, struct madvise_walk_private *private = walk->private; struct mmu_gather *tlb = private->tlb; bool pageout = private->pageout; + bool pageout_anon_only = pageout && !private->can_pageout_file; struct mm_struct *mm = tlb->mm; struct vm_area_struct *vma = walk->vma; pte_t *orig_pte, *pte, ptent; @@ -351,6 +353,9 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, if (page_mapcount(page) != 1) goto huge_unlock; + if (pageout_anon_only && !PageAnon(page)) + goto huge_unlock; + if (next - addr != HPAGE_PMD_SIZE) { int err; @@ -419,6 +424,8 @@ regular_page: if (PageTransCompound(page)) { if (page_mapcount(page) != 1) break; + if (pageout_anon_only && !PageAnon(page)) + break; get_page(page); if (!trylock_page(page)) { put_page(page); @@ -446,6 +453,9 @@ regular_page: if (!allow_shared && (!PageLRU(page) || page_mapcount(page) != 1)) continue; + if (pageout_anon_only && !PageAnon(page)) + continue; + VM_BUG_ON_PAGE(PageTransCompound(page), page); if (pte_young(ptent)) { @@ -499,11 +509,9 @@ static void madvise_cold_page_range(struct mmu_gather *tlb, .tlb = tlb, }; - vm_write_begin(vma); tlb_start_vma(tlb, vma); walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); tlb_end_vma(tlb, vma); - vm_write_end(vma); } static long madvise_cold(struct vm_area_struct *vma, @@ -527,24 +535,22 @@ static long madvise_cold(struct vm_area_struct *vma, static void madvise_pageout_page_range(struct mmu_gather *tlb, struct vm_area_struct *vma, - unsigned long addr, unsigned long end) + unsigned long addr, unsigned long end, + bool can_pageout_file) { struct madvise_walk_private walk_private = { .pageout = true, .tlb = tlb, + .can_pageout_file = can_pageout_file, }; - vm_write_begin(vma); tlb_start_vma(tlb, vma); walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); tlb_end_vma(tlb, vma); - vm_write_end(vma); } -static inline bool can_do_pageout(struct vm_area_struct *vma) +static inline bool can_do_file_pageout(struct vm_area_struct *vma) { - if (vma_is_anonymous(vma)) - return true; if (!vma->vm_file) return false; /* @@ -563,17 +569,23 @@ static long madvise_pageout(struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; struct mmu_gather tlb; + bool can_pageout_file; *prev = vma; if (!can_madv_lru_vma(vma)) return -EINVAL; - if (!can_do_pageout(vma)) - return 0; + /* + * If the VMA belongs to a private file mapping, there can be private + * dirty pages which can be paged out if even this process is neither + * owner nor write capable of the file. Cache the file access check + * here and use it later during page walk. + */ + can_pageout_file = can_do_file_pageout(vma); lru_add_drain(); tlb_gather_mmu(&tlb, mm, start_addr, end_addr); - madvise_pageout_page_range(&tlb, vma, start_addr, end_addr); + madvise_pageout_page_range(&tlb, vma, start_addr, end_addr, can_pageout_file); tlb_finish_mmu(&tlb, start_addr, end_addr); return 0; @@ -741,12 +753,10 @@ static int madvise_free_single_vma(struct vm_area_struct *vma, update_hiwater_rss(mm); mmu_notifier_invalidate_range_start(&range); - vm_write_begin(vma); tlb_start_vma(&tlb, vma); walk_page_range(vma->vm_mm, range.start, range.end, &madvise_free_walk_ops, &tlb); tlb_end_vma(&tlb, vma); - vm_write_end(vma); mmu_notifier_invalidate_range_end(&range); tlb_finish_mmu(&tlb, range.start, range.end); diff --git a/mm/memory.c b/mm/memory.c index 25adbef35900..834078ab6e1c 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1506,7 +1506,6 @@ void unmap_page_range(struct mmu_gather *tlb, unsigned long next; BUG_ON(addr >= end); - vm_write_begin(vma); tlb_start_vma(tlb, vma); pgd = pgd_offset(vma->vm_mm, addr); do { @@ -1516,7 +1515,6 @@ void unmap_page_range(struct mmu_gather *tlb, next = zap_p4d_range(tlb, vma, pgd, addr, next, details); } while (pgd++, addr = next, addr != end); tlb_end_vma(tlb, vma); - vm_write_end(vma); } @@ -3398,6 +3396,8 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) if (userfaultfd_pte_wp(vma, *vmf->pte)) { pte_unmap_unlock(vmf->pte, vmf->ptl); + if (vmf->flags & FAULT_FLAG_SPECULATIVE) + return VM_FAULT_RETRY; return handle_userfault(vmf, VM_UFFD_WP); } @@ -3610,6 +3610,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) vm_fault_t ret; void *shadow = NULL; + if (vmf->flags & FAULT_FLAG_SPECULATIVE) { + pte_unmap(vmf->pte); + return VM_FAULT_RETRY; + } + ret = pte_unmap_same(vmf); if (ret) { /* @@ -3869,6 +3874,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) if (vmf->vma_flags & VM_SHARED) return VM_FAULT_SIGBUS; + /* Do not check unstable pmd, if it's changed will retry later */ + if (vmf->flags & FAULT_FLAG_SPECULATIVE) + goto skip_pmd_checks; + /* * Use pte_alloc() instead of pte_alloc_map(). We can't run * pte_offset_map() on pmds where a huge pmd might be created @@ -3886,6 +3895,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) if (unlikely(pmd_trans_unstable(vmf->pmd))) return 0; +skip_pmd_checks: /* Use the zero-page for reads */ if (!(vmf->flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(vma->vm_mm)) { @@ -3993,6 +4003,10 @@ static vm_fault_t __do_fault(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; vm_fault_t ret; + /* Do not check unstable pmd, if it's changed will retry later */ + if (vmf->flags & FAULT_FLAG_SPECULATIVE) + goto skip_pmd_checks; + /* * Preallocate pte before we take page_lock because this might lead to * deadlocks for memcg reclaim which waits for pages under writeback: @@ -4015,6 +4029,7 @@ static vm_fault_t __do_fault(struct vm_fault *vmf) smp_wmb(); /* See comment in __pte_alloc() */ } +skip_pmd_checks: ret = vma->vm_ops->fault(vmf); if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY | VM_FAULT_DONE_COW))) @@ -4190,7 +4205,11 @@ vm_fault_t finish_fault(struct vm_fault *vmf) return ret; } - if (pmd_none(*vmf->pmd) && !(vmf->flags & FAULT_FLAG_SPECULATIVE)) { + /* Do not check unstable pmd, if it's changed will retry later */ + if (vmf->flags & FAULT_FLAG_SPECULATIVE) + goto skip_pmd_checks; + + if (pmd_none(*vmf->pmd)) { if (PageTransCompound(page)) { ret = do_set_pmd(vmf, page); if (ret != VM_FAULT_FALLBACK) @@ -4210,10 +4229,14 @@ vm_fault_t finish_fault(struct vm_fault *vmf) } } - /* See comment in handle_pte_fault() */ + /* + * See comment in handle_pte_fault() for how this scenario happens, we + * need to return NOPAGE so that we drop this page. + */ if (pmd_devmap_trans_unstable(vmf->pmd)) - return 0; + return VM_FAULT_NOPAGE; +skip_pmd_checks: if (!pte_map_lock(vmf)) return VM_FAULT_RETRY; @@ -4313,7 +4336,8 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1, start_pgoff + nr_pages - 1); - if (pmd_none(*vmf->pmd)) { + if (!(vmf->flags & FAULT_FLAG_SPECULATIVE) && + pmd_none(*vmf->pmd)) { vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm); if (!vmf->prealloc_pte) return VM_FAULT_OOM; @@ -4671,16 +4695,11 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) pte_t entry; vm_fault_t ret = 0; + /* Do not check unstable pmd, if it's changed will retry later */ + if (vmf->flags & FAULT_FLAG_SPECULATIVE) + goto skip_pmd_checks; + if (unlikely(pmd_none(*vmf->pmd))) { - /* - * In the case of the speculative page fault handler we abort - * the speculative path immediately as the pmd is probably - * in the way to be converted in a huge one. We will try - * again holding the mmap_sem (which implies that the collapse - * operation is done). - */ - if (vmf->flags & FAULT_FLAG_SPECULATIVE) - return VM_FAULT_RETRY; /* * Leave __pte_alloc() until later: because vm_ops->fault may * want to allocate huge page, and if we expose page table @@ -4688,7 +4707,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) * concurrent faults and from rmap lookups. */ vmf->pte = NULL; - } else if (!(vmf->flags & FAULT_FLAG_SPECULATIVE)) { + } else { /* * If a huge pmd materialized under us just retry later. Use * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead @@ -4730,6 +4749,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) } } +skip_pmd_checks: if (!vmf->pte) { if (vma_is_anonymous(vmf->vma)) return do_anonymous_page(vmf); @@ -5012,11 +5032,13 @@ static vm_fault_t ___handle_speculative_fault(struct mm_struct *mm, vmf.vma_flags = READ_ONCE(vmf.vma->vm_flags); vmf.vma_page_prot = READ_ONCE(vmf.vma->vm_page_prot); +#ifdef CONFIG_USERFAULTFD /* Can't call userland page fault handler in the speculative path */ - if (unlikely(vmf.vma_flags & VM_UFFD_MISSING)) { + if (unlikely(vmf.vma_flags & __VM_UFFD_FLAGS)) { trace_spf_vma_notsup(_RET_IP_, vmf.vma, address); return VM_FAULT_RETRY; } +#endif if (vmf.vma_flags & VM_GROWSDOWN || vmf.vma_flags & VM_GROWSUP) { /* @@ -5055,11 +5077,10 @@ static vm_fault_t ___handle_speculative_fault(struct mm_struct *mm, pol = __get_vma_policy(vmf.vma, address); if (!pol) pol = get_task_policy(current); - if (!pol) - if (pol && pol->mode == MPOL_INTERLEAVE) { - trace_spf_vma_notsup(_RET_IP_, vmf.vma, address); - return VM_FAULT_RETRY; - } + if (pol && pol->mode == MPOL_INTERLEAVE) { + trace_spf_vma_notsup(_RET_IP_, vmf.vma, address); + return VM_FAULT_RETRY; + } #endif /* diff --git a/mm/mempolicy.c b/mm/mempolicy.c index c4c1c7bf51d7..142929e67a97 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -657,11 +657,9 @@ unsigned long change_prot_numa(struct vm_area_struct *vma, { int nr_updated; - vm_write_begin(vma); nr_updated = change_protection(vma, addr, end, PAGE_NONE, MM_CP_PROT_NUMA); if (nr_updated) count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated); - vm_write_end(vma); return nr_updated; } diff --git a/mm/mmap.c b/mm/mmap.c index d4ee5c4880c5..8ec92cc5ae69 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2424,8 +2424,22 @@ struct vm_area_struct *get_vma(struct mm_struct *mm, unsigned long addr) read_lock(&mm->mm_rb_lock); vma = __find_vma(mm, addr); - if (vma) - atomic_inc(&vma->vm_ref_count); + + /* + * If there is a concurrent fast mremap, bail out since the entire + * PMD/PUD subtree may have been remapped. + * + * This is usually safe for conventional mremap since it takes the + * PTE locks as does SPF. However fast mremap only takes the lock + * at the PMD/PUD level which is ok as it is done with the mmap + * write lock held. But since SPF, as the term implies forgoes, + * taking the mmap read lock and also cannot take PTL lock at the + * larger PMD/PUD granualrity, since it would introduce huge + * contention in the page fault path; fall back to regular fault + * handling. + */ + if (vma && !atomic_inc_unless_negative(&vma->vm_ref_count)) + vma = NULL; read_unlock(&mm->mm_rb_lock); return vma; @@ -3349,6 +3363,7 @@ void exit_mmap(struct mm_struct *mm) vma = remove_vma(vma); cond_resched(); } + mm->mmap = NULL; mmap_write_unlock(mm); vm_unacct_memory(nr_accounted); } diff --git a/mm/mremap.c b/mm/mremap.c index 07f7c869a38f..b1263e9a16af 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -210,6 +210,38 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, drop_rmap_locks(vma); } +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT +static inline bool trylock_vma_ref_count(struct vm_area_struct *vma) +{ + /* + * If we have the only reference, swap the refcount to -1. This + * will prevent other concurrent references by get_vma() for SPFs. + */ + return atomic_cmpxchg(&vma->vm_ref_count, 1, -1) == 1; +} + +/* + * Restore the VMA reference count to 1 after a fast mremap. + */ +static inline void unlock_vma_ref_count(struct vm_area_struct *vma) +{ + /* + * This should only be called after a corresponding, + * successful trylock_vma_ref_count(). + */ + VM_BUG_ON_VMA(atomic_cmpxchg(&vma->vm_ref_count, -1, 1) != -1, + vma); +} +#else /* !CONFIG_SPECULATIVE_PAGE_FAULT */ +static inline bool trylock_vma_ref_count(struct vm_area_struct *vma) +{ + return true; +} +static inline void unlock_vma_ref_count(struct vm_area_struct *vma) +{ +} +#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */ + #ifdef CONFIG_HAVE_MOVE_PMD static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr, unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd) @@ -244,6 +276,14 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr, if (WARN_ON_ONCE(!pmd_none(*new_pmd))) return false; + /* + * We hold both exclusive mmap_lock and rmap_lock at this point and + * cannot block. If we cannot immediately take exclusive ownership + * of the VMA fallback to the move_ptes(). + */ + if (!trylock_vma_ref_count(vma)) + return false; + /* * We don't have to worry about the ordering of src and dst * ptlocks because exclusive mmap_lock prevents deadlock. @@ -266,6 +306,7 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr, spin_unlock(new_ptl); spin_unlock(old_ptl); + unlock_vma_ref_count(vma); return true; } #else @@ -292,6 +333,14 @@ static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr, if (WARN_ON_ONCE(!pud_none(*new_pud))) return false; + /* + * We hold both exclusive mmap_lock and rmap_lock at this point and + * cannot block. If we cannot immediately take exclusive ownership + * of the VMA fallback to the move_ptes(). + */ + if (!trylock_vma_ref_count(vma)) + return false; + /* * We don't have to worry about the ordering of src and dst * ptlocks because exclusive mmap_lock prevents deadlock. @@ -314,6 +363,7 @@ static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr, spin_unlock(new_ptl); spin_unlock(old_ptl); + unlock_vma_ref_count(vma); return true; } #else diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c916948b873b..18a969923841 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4956,6 +4956,12 @@ retry: if (current->flags & PF_MEMALLOC) goto nopage; + trace_android_vh_alloc_pages_reclaim_bypass(gfp_mask, order, + alloc_flags, ac->migratetype, &page); + + if (page) + goto got_pg; + /* Try direct reclaim and then allocating */ page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, &did_some_progress); @@ -5071,6 +5077,11 @@ nopage: goto retry; } fail: + trace_android_vh_alloc_pages_failure_bypass(gfp_mask, order, + alloc_flags, ac->migratetype, &page); + if (page) + goto got_pg; + warn_alloc(gfp_mask, ac->nodemask, "page allocation failure: order:%u", order); got_pg: @@ -6412,7 +6423,6 @@ static void __meminit zone_init_free_lists(struct zone *zone) } } -#if !defined(CONFIG_FLAT_NODE_MEM_MAP) /* * Only struct pages that correspond to ranges defined by memblock.memory * are zeroed and initialized by going through __init_single_page() during @@ -6457,13 +6467,6 @@ static void __init init_unavailable_range(unsigned long spfn, pr_info("On node %d, zone %s: %lld pages in unavailable ranges", node, zone_names[zone], pgcnt); } -#else -static inline void init_unavailable_range(unsigned long spfn, - unsigned long epfn, - int zone, int node) -{ -} -#endif static void __init memmap_init_zone_range(struct zone *zone, unsigned long start_pfn, diff --git a/mm/slab.c b/mm/slab.c index eced754636ad..aa4ef18ddfb6 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3016,10 +3016,9 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, objp += obj_offset(cachep); if (cachep->ctor && cachep->flags & SLAB_POISON) cachep->ctor(objp); - if (ARCH_SLAB_MINALIGN && - ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) { - pr_err("0x%px: not aligned to ARCH_SLAB_MINALIGN=%d\n", - objp, (int)ARCH_SLAB_MINALIGN); + if ((unsigned long)objp & (arch_slab_minalign() - 1)) { + pr_err("0x%px: not aligned to arch_slab_minalign()=%u\n", objp, + arch_slab_minalign()); } return objp; } @@ -3428,6 +3427,7 @@ static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp, if (is_kfence_address(objp)) { kmemleak_free_recursive(objp, cachep->flags); + memcg_slab_free_hook(cachep, &objp, 1); __kfence_free(objp); return; } diff --git a/mm/slab_common.c b/mm/slab_common.c index c751b18f7e60..05135ebb6159 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -155,8 +155,7 @@ static unsigned int calculate_alignment(slab_flags_t flags, align = max(align, ralign); } - if (align < ARCH_SLAB_MINALIGN) - align = ARCH_SLAB_MINALIGN; + align = max(align, arch_slab_minalign()); return ALIGN(align, sizeof(void *)); } diff --git a/mm/slob.c b/mm/slob.c index 7cc9805c8091..37072a7d2f42 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -469,9 +469,11 @@ static __always_inline void * __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) { unsigned int *m; - int minalign = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); + unsigned int minalign; void *ret; + minalign = max_t(unsigned int, ARCH_KMALLOC_MINALIGN, + arch_slab_minalign()); gfp &= gfp_allowed_mask; fs_reclaim_acquire(gfp); @@ -485,7 +487,7 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) * kmalloc()'d objects. */ if (is_power_of_2(size)) - align = max(minalign, (int) size); + align = max_t(unsigned int, minalign, size); if (!size) return ZERO_SIZE_PTR; @@ -547,8 +549,11 @@ void kfree(const void *block) sp = virt_to_page(block); if (PageSlab(sp)) { - int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); + unsigned int align = max_t(unsigned int, + ARCH_KMALLOC_MINALIGN, + arch_slab_minalign()); unsigned int *m = (unsigned int *)(block - align); + slob_free(m, *m + align); } else { unsigned int order = compound_order(sp); @@ -564,7 +569,7 @@ EXPORT_SYMBOL(kfree); size_t __ksize(const void *block) { struct page *sp; - int align; + unsigned int align; unsigned int *m; BUG_ON(!block); @@ -575,7 +580,8 @@ size_t __ksize(const void *block) if (unlikely(!PageSlab(sp))) return page_size(sp); - align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); + align = max_t(unsigned int, ARCH_KMALLOC_MINALIGN, + arch_slab_minalign()); m = (unsigned int *)(block - align); return SLOB_UNITS(*m) * SLOB_UNIT; } diff --git a/net/core/skbuff.c b/net/core/skbuff.c index d08b0216717b..382dbdc553bc 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1676,11 +1676,10 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, skb->head = data; skb->head_frag = 0; skb->data += off; + + skb_set_end_offset(skb, size); #ifdef NET_SKBUFF_DATA_USES_OFFSET - skb->end = size; off = nhead; -#else - skb->end = skb->head + size; #endif skb->tail += off; skb_headers_offset_update(skb, nhead); @@ -1728,6 +1727,38 @@ struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) } EXPORT_SYMBOL(skb_realloc_headroom); +int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) +{ + unsigned int saved_end_offset, saved_truesize; + struct skb_shared_info *shinfo; + int res; + + saved_end_offset = skb_end_offset(skb); + saved_truesize = skb->truesize; + + res = pskb_expand_head(skb, 0, 0, pri); + if (res) + return res; + + skb->truesize = saved_truesize; + + if (likely(skb_end_offset(skb) == saved_end_offset)) + return 0; + + shinfo = skb_shinfo(skb); + + /* We are about to change back skb->end, + * we need to move skb_shinfo() to its new location. + */ + memmove(skb->head + saved_end_offset, + shinfo, + offsetof(struct skb_shared_info, frags[shinfo->nr_frags])); + + skb_set_end_offset(skb, saved_end_offset); + + return 0; +} + /** * skb_copy_expand - copy and expand sk_buff * @skb: buffer to copy @@ -5976,11 +6007,7 @@ static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, skb->head = data; skb->data = data; skb->head_frag = 0; -#ifdef NET_SKBUFF_DATA_USES_OFFSET - skb->end = size; -#else - skb->end = skb->head + size; -#endif + skb_set_end_offset(skb, size); skb_set_tail_pointer(skb, skb_headlen(skb)); skb_headers_offset_update(skb, 0); skb->cloned = 0; @@ -6118,11 +6145,7 @@ static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, skb->head = data; skb->head_frag = 0; skb->data = data; -#ifdef NET_SKBUFF_DATA_USES_OFFSET - skb->end = size; -#else - skb->end = skb->head + size; -#endif + skb_set_end_offset(skb, size); skb_reset_tail_pointer(skb); skb_headers_offset_update(skb, 0); skb->cloned = 0; diff --git a/samples/crypto/fips140_lab_util.c b/samples/crypto/fips140_lab_util.c index 996839dbd2e3..5f8e9018013a 100644 --- a/samples/crypto/fips140_lab_util.c +++ b/samples/crypto/fips140_lab_util.c @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -45,6 +46,8 @@ * ---------------------------------------------------------------------------*/ #define ARRAY_SIZE(A) (sizeof(A) / sizeof((A)[0])) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define MAX(a, b) ((a) > (b) ? (a) : (b)) static void __attribute__((noreturn)) do_die(const char *format, va_list va, int err) @@ -109,6 +112,23 @@ static const char *bytes_to_hex(const uint8_t *bytes, size_t count) return hex; } +static void full_write(int fd, const void *buf, size_t count) +{ + while (count) { + ssize_t ret = write(fd, buf, count); + + if (ret < 0) + die_errno("write failed"); + buf += ret; + count -= ret; + } +} + +enum { + OPT_AMOUNT, + OPT_ITERATIONS, +}; + static void usage(void); /* --------------------------------------------------------------------------- @@ -226,6 +246,68 @@ static int get_req_fd(int alg_fd, const char *alg_name) return req_fd; } +/* --------------------------------------------------------------------------- + * dump_jitterentropy command + * ---------------------------------------------------------------------------*/ + +static void dump_from_jent_fd(int fd, size_t count) +{ + uint8_t buf[AF_ALG_MAX_RNG_REQUEST_SIZE]; + + while (count) { + ssize_t ret; + + memset(buf, 0, sizeof(buf)); + ret = read(fd, buf, MIN(count, sizeof(buf))); + if (ret < 0) + die_errno("error reading from jitterentropy_rng"); + full_write(STDOUT_FILENO, buf, ret); + count -= ret; + } +} + +static int cmd_dump_jitterentropy(int argc, char *argv[]) +{ + static const struct option longopts[] = { + { "amount", required_argument, NULL, OPT_AMOUNT }, + { "iterations", required_argument, NULL, OPT_ITERATIONS }, + { NULL, 0, NULL, 0 }, + }; + size_t amount = 128; + size_t iterations = 1; + size_t i; + int c; + + while ((c = getopt_long(argc, argv, "", longopts, NULL)) != -1) { + switch (c) { + case OPT_AMOUNT: + amount = strtoul(optarg, NULL, 0); + if (amount <= 0 || amount >= ULONG_MAX) + die("invalid argument to --amount"); + break; + case OPT_ITERATIONS: + iterations = strtoul(optarg, NULL, 0); + if (iterations <= 0 || iterations >= ULONG_MAX) + die("invalid argument to --iterations"); + break; + default: + usage(); + return 1; + } + } + + for (i = 0; i < iterations; i++) { + int alg_fd = get_alg_fd("rng", "jitterentropy_rng"); + int req_fd = get_req_fd(alg_fd, "jitterentropy_rng"); + + dump_from_jent_fd(req_fd, amount); + + close(req_fd); + close(alg_fd); + } + return 0; +} + /* --------------------------------------------------------------------------- * show_invalid_inputs command * ---------------------------------------------------------------------------*/ @@ -510,6 +592,7 @@ static const struct command { const char *name; int (*func)(int argc, char *argv[]); } commands[] = { + { "dump_jitterentropy", cmd_dump_jitterentropy }, { "show_invalid_inputs", cmd_show_invalid_inputs }, { "show_module_version", cmd_show_module_version }, { "show_service_indicators", cmd_show_service_indicators }, @@ -519,9 +602,14 @@ static void usage(void) { fprintf(stderr, "Usage:\n" +" fips140_lab_util dump_jitterentropy [OPTION]...\n" " fips140_lab_util show_invalid_inputs\n" " fips140_lab_util show_module_version\n" " fips140_lab_util show_service_indicators [SERVICE]...\n" +"\n" +"Options for dump_jitterentropy:\n" +" --amount=AMOUNT Amount to dump in bytes per iteration (default 128)\n" +" --iterations=COUNT Number of start-up iterations (default 1)\n" ); } diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 4963ee9f218b..1a4aa5edfd1e 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -388,7 +388,7 @@ ifeq ($(CONFIG_LTO_CLANG) $(CONFIG_MODVERSIONS),y y) cmd_update_lto_symversions = \ rm -f $@.symversions \ $(foreach n, $(filter-out FORCE,$^), \ - $(if $(wildcard $(n).symversions), \ + $(if $(shell test -s $(n).symversions && echo y), \ ; cat $(n).symversions >> $@.symversions)) else cmd_update_lto_symversions = echo >/dev/null diff --git a/scripts/adjust_autoksyms.sh b/scripts/adjust_autoksyms.sh index 2b366d945ccb..d8f6f9c63043 100755 --- a/scripts/adjust_autoksyms.sh +++ b/scripts/adjust_autoksyms.sh @@ -34,9 +34,6 @@ case "$KBUILD_VERBOSE" in ;; esac -# We need access to CONFIG_ symbols -. include/config/auto.conf - # Generate a new symbol list file $CONFIG_SHELL $srctree/scripts/gen_autoksyms.sh "$new_ksyms_file" diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index d9c283503159..5074c8b82367 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -9,6 +9,7 @@ TARGETS += clone3 TARGETS += core TARGETS += cpufreq TARGETS += cpu-hotplug +TARGETS += damon TARGETS += drivers/dma-buf TARGETS += efivarfs TARGETS += exec diff --git a/tools/vm/slabinfo.c b/tools/vm/slabinfo.c index 3ae985dc24b6..0fffaeedee76 100644 --- a/tools/vm/slabinfo.c +++ b/tools/vm/slabinfo.c @@ -233,6 +233,24 @@ static unsigned long read_slab_obj(struct slabinfo *s, const char *name) return l; } +static unsigned long read_debug_slab_obj(struct slabinfo *s, const char *name) +{ + char x[128]; + FILE *f; + size_t l; + + snprintf(x, 128, "/sys/kernel/debug/slab/%s/%s", s->name, name); + f = fopen(x, "r"); + if (!f) { + buffer[0] = 0; + l = 0; + } else { + l = fread(buffer, 1, sizeof(buffer), f); + buffer[l] = 0; + fclose(f); + } + return l; +} /* * Put a size string together @@ -409,14 +427,18 @@ static void show_tracking(struct slabinfo *s) { printf("\n%s: Kernel object allocation\n", s->name); printf("-----------------------------------------------------------------------\n"); - if (read_slab_obj(s, "alloc_calls")) + if (read_debug_slab_obj(s, "alloc_traces")) + printf("%s", buffer); + else if (read_slab_obj(s, "alloc_calls")) printf("%s", buffer); else printf("No Data\n"); printf("\n%s: Kernel object freeing\n", s->name); printf("------------------------------------------------------------------------\n"); - if (read_slab_obj(s, "free_calls")) + if (read_debug_slab_obj(s, "free_traces")) + printf("%s", buffer); + else if (read_slab_obj(s, "free_calls")) printf("%s", buffer); else printf("No Data\n");