From 49971e6bad2da980686368baa34039907f62b516 Mon Sep 17 00:00:00 2001 From: Olaf Hering Date: Wed, 19 Aug 2020 11:05:09 +0200 Subject: [PATCH 001/265] drivers: hv: remove cast from hyperv_die_event No need to cast a void pointer. Signed-off-by: Olaf Hering Link: https://lore.kernel.org/r/20200819090510.28995-1-olaf@aepfle.de Signed-off-by: Wei Liu --- drivers/hv/vmbus_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 910b6e90866c..187809977360 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -83,7 +83,7 @@ static int hyperv_panic_event(struct notifier_block *nb, unsigned long val, static int hyperv_die_event(struct notifier_block *nb, unsigned long val, void *args) { - struct die_args *die = (struct die_args *)args; + struct die_args *die = args; struct pt_regs *regs = die->regs; /* Don't notify Hyper-V if the die event is other than oops */ From 72f80ce4ef9b756185aab5f1955d8352b6021fba Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 13 Aug 2020 08:38:37 +0530 Subject: [PATCH 002/265] opp: Rename regulator_enabled and use it as status of all resources Expand the scope of the regulator_enabled flag and use it to track status of all the resources. This will be used for other stuff in the next patch. Tested-by: Rajendra Nayak Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 19 +++++++++---------- drivers/opp/opp.h | 4 ++-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 9668ea04cc80..6f43ef4945b7 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -703,12 +703,10 @@ static int _generic_set_opp_regulator(struct opp_table *opp_table, * Enable the regulator after setting its voltages, otherwise it breaks * some boot-enabled regulators. */ - if (unlikely(!opp_table->regulator_enabled)) { + if (unlikely(!opp_table->enabled)) { ret = regulator_enable(reg); if (ret < 0) dev_warn(dev, "Failed to enable regulator: %d", ret); - else - opp_table->regulator_enabled = true; } return 0; @@ -909,12 +907,12 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) if (ret) goto put_opp_table; - if (opp_table->regulator_enabled) { + if (opp_table->regulators) regulator_disable(opp_table->regulators[0]); - opp_table->regulator_enabled = false; - } ret = _set_required_opps(dev, opp_table, NULL); + + opp_table->enabled = false; goto put_opp_table; } @@ -1001,8 +999,11 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) dev_err(dev, "Failed to set required opps: %d\n", ret); } - if (!ret) + if (!ret) { ret = _set_opp_bw(opp_table, opp, dev, false); + if (!ret) + opp_table->enabled = true; + } put_opp: dev_pm_opp_put(opp); @@ -1796,11 +1797,9 @@ void dev_pm_opp_put_regulators(struct opp_table *opp_table) /* Make sure there are no concurrent readers while updating opp_table */ WARN_ON(!list_empty(&opp_table->opp_list)); - if (opp_table->regulator_enabled) { + if (opp_table->enabled) { for (i = opp_table->regulator_count - 1; i >= 0; i--) regulator_disable(opp_table->regulators[i]); - - opp_table->regulator_enabled = false; } for (i = opp_table->regulator_count - 1; i >= 0; i--) diff --git a/drivers/opp/opp.h b/drivers/opp/opp.h index e51646ff279e..0c3de3f6db5c 100644 --- a/drivers/opp/opp.h +++ b/drivers/opp/opp.h @@ -147,11 +147,11 @@ enum opp_table_access { * @clk: Device's clock handle * @regulators: Supply regulators * @regulator_count: Number of power supply regulators. Its value can be -1 - * @regulator_enabled: Set to true if regulators were previously enabled. * (uninitialized), 0 (no opp-microvolt property) or > 0 (has opp-microvolt * property). * @paths: Interconnect path handles * @path_count: Number of interconnect paths + * @enabled: Set to true if the device's resources are enabled/configured. * @genpd_performance_state: Device's power domain support performance state. * @is_genpd: Marks if the OPP table belongs to a genpd. * @set_opp: Platform specific set_opp callback @@ -195,9 +195,9 @@ struct opp_table { struct clk *clk; struct regulator **regulators; int regulator_count; - bool regulator_enabled; struct icc_path **paths; unsigned int path_count; + bool enabled; bool genpd_performance_state; bool is_genpd; From 10b217365b9454bc5f31d2801148fdf04939f431 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 13 Aug 2020 08:38:37 +0530 Subject: [PATCH 003/265] opp: Reuse the enabled flag in !target_freq path The OPP core needs to track if the resources of devices are enabled/configured or not, as it disables the resources when target_freq is set to 0. Handle that with the new enabled flag and remove otherwise complex conditional statements. Tested-by: Rajendra Nayak Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 6f43ef4945b7..0b437d483b75 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -886,22 +886,18 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) } if (unlikely(!target_freq)) { + ret = 0; + + if (!opp_table->enabled) + goto put_opp_table; + /* * Some drivers need to support cases where some platforms may * have OPP table for the device, while others don't and * opp_set_rate() just needs to behave like clk_set_rate(). */ - if (!_get_opp_count(opp_table)) { - ret = 0; + if (!_get_opp_count(opp_table)) goto put_opp_table; - } - - if (!opp_table->required_opp_tables && !opp_table->regulators && - !opp_table->paths) { - dev_err(dev, "target frequency can't be 0\n"); - ret = -EINVAL; - goto put_opp_table; - } ret = _set_opp_bw(opp_table, NULL, dev, true); if (ret) @@ -931,14 +927,11 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) old_freq = clk_get_rate(clk); /* Return early if nothing to do */ - if (old_freq == freq) { - if (!opp_table->required_opp_tables && !opp_table->regulators && - !opp_table->paths) { - dev_dbg(dev, "%s: old/new frequencies (%lu Hz) are same, nothing to do\n", - __func__, freq); - ret = 0; - goto put_opp_table; - } + if (opp_table->enabled && old_freq == freq) { + dev_dbg(dev, "%s: old/new frequencies (%lu Hz) are same, nothing to do\n", + __func__, freq); + ret = 0; + goto put_opp_table; } /* From f3364e17d5716a7356f16fe73c36cba50633ee18 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 13 Aug 2020 09:48:04 +0530 Subject: [PATCH 004/265] opp: Split out _opp_set_rate_zero() Create separate routine _opp_set_rate_zero() to handle !target_freq case. Tested-by: Rajendra Nayak Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 52 ++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 0b437d483b75..4edd2c3d6d91 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -860,6 +860,34 @@ int dev_pm_opp_set_bw(struct device *dev, struct dev_pm_opp *opp) } EXPORT_SYMBOL_GPL(dev_pm_opp_set_bw); +static int _opp_set_rate_zero(struct device *dev, struct opp_table *opp_table) +{ + int ret; + + if (!opp_table->enabled) + return 0; + + /* + * Some drivers need to support cases where some platforms may + * have OPP table for the device, while others don't and + * opp_set_rate() just needs to behave like clk_set_rate(). + */ + if (!_get_opp_count(opp_table)) + return 0; + + ret = _set_opp_bw(opp_table, NULL, dev, true); + if (ret) + return ret; + + if (opp_table->regulators) + regulator_disable(opp_table->regulators[0]); + + ret = _set_required_opps(dev, opp_table, NULL); + + opp_table->enabled = false; + return ret; +} + /** * dev_pm_opp_set_rate() - Configure new OPP based on frequency * @dev: device for which we do this operation @@ -886,29 +914,7 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) } if (unlikely(!target_freq)) { - ret = 0; - - if (!opp_table->enabled) - goto put_opp_table; - - /* - * Some drivers need to support cases where some platforms may - * have OPP table for the device, while others don't and - * opp_set_rate() just needs to behave like clk_set_rate(). - */ - if (!_get_opp_count(opp_table)) - goto put_opp_table; - - ret = _set_opp_bw(opp_table, NULL, dev, true); - if (ret) - goto put_opp_table; - - if (opp_table->regulators) - regulator_disable(opp_table->regulators[0]); - - ret = _set_required_opps(dev, opp_table, NULL); - - opp_table->enabled = false; + ret = _opp_set_rate_zero(dev, opp_table); goto put_opp_table; } From 8aaf6264fc7fde15864c3e4d84ccc8b3af6811f3 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 20 Aug 2020 13:18:23 +0530 Subject: [PATCH 005/265] opp: Remove _dev_pm_opp_find_and_remove_table() wrapper Remove the unnecessary wrapper and merge _dev_pm_opp_find_and_remove_table() with dev_pm_opp_remove_table(). Tested-by: Rajendra Nayak Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 21 ++++++++------------- drivers/opp/cpu.c | 2 +- drivers/opp/of.c | 2 +- drivers/opp/opp.h | 1 - 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 4edd2c3d6d91..6978b9218c6e 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2395,7 +2395,14 @@ int dev_pm_opp_unregister_notifier(struct device *dev, } EXPORT_SYMBOL(dev_pm_opp_unregister_notifier); -void _dev_pm_opp_find_and_remove_table(struct device *dev) +/** + * dev_pm_opp_remove_table() - Free all OPPs associated with the device + * @dev: device pointer used to lookup OPP table. + * + * Free both OPPs created using static entries present in DT and the + * dynamically added entries. + */ +void dev_pm_opp_remove_table(struct device *dev) { struct opp_table *opp_table; @@ -2420,16 +2427,4 @@ void _dev_pm_opp_find_and_remove_table(struct device *dev) /* Drop reference taken while the OPP table was added */ dev_pm_opp_put_opp_table(opp_table); } - -/** - * dev_pm_opp_remove_table() - Free all OPPs associated with the device - * @dev: device pointer used to lookup OPP table. - * - * Free both OPPs created using static entries present in DT and the - * dynamically added entries. - */ -void dev_pm_opp_remove_table(struct device *dev) -{ - _dev_pm_opp_find_and_remove_table(dev); -} EXPORT_SYMBOL_GPL(dev_pm_opp_remove_table); diff --git a/drivers/opp/cpu.c b/drivers/opp/cpu.c index b5055cc886ef..5004335cf0de 100644 --- a/drivers/opp/cpu.c +++ b/drivers/opp/cpu.c @@ -124,7 +124,7 @@ void _dev_pm_opp_cpumask_remove_table(const struct cpumask *cpumask, continue; } - _dev_pm_opp_find_and_remove_table(cpu_dev); + dev_pm_opp_remove_table(cpu_dev); } } diff --git a/drivers/opp/of.c b/drivers/opp/of.c index 0430290670ab..7d9d4455a59e 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -616,7 +616,7 @@ free_microvolt: */ void dev_pm_opp_of_remove_table(struct device *dev) { - _dev_pm_opp_find_and_remove_table(dev); + dev_pm_opp_remove_table(dev); } EXPORT_SYMBOL_GPL(dev_pm_opp_of_remove_table); diff --git a/drivers/opp/opp.h b/drivers/opp/opp.h index 0c3de3f6db5c..78e876ec803e 100644 --- a/drivers/opp/opp.h +++ b/drivers/opp/opp.h @@ -217,7 +217,6 @@ void _get_opp_table_kref(struct opp_table *opp_table); int _get_opp_count(struct opp_table *opp_table); struct opp_table *_find_opp_table(struct device *dev); struct opp_device *_add_opp_dev(const struct device *dev, struct opp_table *opp_table); -void _dev_pm_opp_find_and_remove_table(struct device *dev); struct dev_pm_opp *_opp_allocate(struct opp_table *opp_table); void _opp_free(struct dev_pm_opp *opp); int _opp_compare_key(struct dev_pm_opp *opp1, struct dev_pm_opp *opp2); From dd461cd9183fe80bee84fd6cab2a29bfc8f55a76 Mon Sep 17 00:00:00 2001 From: Stephan Gerhold Date: Mon, 27 Jul 2020 11:30:46 +0200 Subject: [PATCH 006/265] opp: Allow dev_pm_opp_get_opp_table() to return -EPROBE_DEFER The OPP core manages various resources, e.g. clocks or interconnect paths. These resources are looked up when the OPP table is allocated once dev_pm_opp_get_opp_table() is called the first time (either directly or indirectly through one of the many helper functions). At this point, the resources may not be available yet, i.e. looking them up will result in -EPROBE_DEFER. Unfortunately, dev_pm_opp_get_opp_table() is currently unable to propagate this error code since it only returns the allocated OPP table or NULL. This means that all consumers of the OPP core are required to make sure that all necessary resources are available. Usually this happens by requesting them, checking the result and releasing them immediately after. For example, we have added "dev_pm_opp_of_find_icc_paths(dev, NULL)" to several drivers now just to make sure the interconnect providers are ready before the OPP table is allocated. If this call is missing, the OPP core will only warn about this and then attempt to continue without interconnect. This will eventually fail horribly, e.g.: cpu cpu0: _allocate_opp_table: Error finding interconnect paths: -517 ... later ... of: _read_bw: Mismatch between opp-peak-kBps and paths (1 0) cpu cpu0: _opp_add_static_v2: opp key field not found cpu cpu0: _of_add_opp_table_v2: Failed to add OPP, -22 This example happens when trying to use interconnects for a CPU OPP table together with qcom-cpufreq-nvmem.c. qcom-cpufreq-nvmem calls dev_pm_opp_set_supported_hw(), which ends up allocating the OPP table early. To fix the problem with the current approach we would need to add yet another call to dev_pm_opp_of_find_icc_paths(dev, NULL). But actually qcom-cpufreq-nvmem.c has nothing to do with interconnects... This commit attempts to make this more robust by allowing dev_pm_opp_get_opp_table() to return an error pointer. Fixing all the usages is trivial because the function is usually used indirectly through another helper (e.g. dev_pm_opp_set_supported_hw() above). These other helpers already return an error pointer. The example above then works correctly because set_supported_hw() will return -EPROBE_DEFER, and qcom-cpufreq-nvmem.c already propagates that error. It should also be possible to remove the remaining usages of "dev_pm_opp_of_find_icc_paths(dev, NULL)" from other drivers as well. Note that this commit currently only handles -EPROBE_DEFER for the clock/interconnects within _allocate_opp_table(). Other errors are just ignored as before. Eventually those should be propagated as well. Signed-off-by: Stephan Gerhold Acked-by: Krzysztof Kozlowski Reviewed-by: Ulf Hansson [ Viresh: skip checking return value of dev_pm_opp_get_opp_table() for EPROBE_DEFER in domain.c, fix NULL return value and reorder code a bit in core.c, and update exynos-asv.c ] Signed-off-by: Viresh Kumar --- drivers/base/power/domain.c | 14 +++++---- drivers/opp/core.c | 53 +++++++++++++++++++------------- drivers/opp/of.c | 8 ++--- drivers/soc/samsung/exynos-asv.c | 2 +- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 2cb5e04cf86c..b92bb61550d3 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -2044,8 +2044,9 @@ int of_genpd_add_provider_simple(struct device_node *np, if (genpd->set_performance_state) { ret = dev_pm_opp_of_add_table(&genpd->dev); if (ret) { - dev_err(&genpd->dev, "Failed to add OPP table: %d\n", - ret); + if (ret != -EPROBE_DEFER) + dev_err(&genpd->dev, "Failed to add OPP table: %d\n", + ret); goto unlock; } @@ -2054,7 +2055,7 @@ int of_genpd_add_provider_simple(struct device_node *np, * state. */ genpd->opp_table = dev_pm_opp_get_opp_table(&genpd->dev); - WARN_ON(!genpd->opp_table); + WARN_ON(IS_ERR(genpd->opp_table)); } ret = genpd_add_provider(np, genpd_xlate_simple, genpd); @@ -2111,8 +2112,9 @@ int of_genpd_add_provider_onecell(struct device_node *np, if (genpd->set_performance_state) { ret = dev_pm_opp_of_add_table_indexed(&genpd->dev, i); if (ret) { - dev_err(&genpd->dev, "Failed to add OPP table for index %d: %d\n", - i, ret); + if (ret != -EPROBE_DEFER) + dev_err(&genpd->dev, "Failed to add OPP table for index %d: %d\n", + i, ret); goto error; } @@ -2121,7 +2123,7 @@ int of_genpd_add_provider_onecell(struct device_node *np, * performance state. */ genpd->opp_table = dev_pm_opp_get_opp_table_indexed(&genpd->dev, i); - WARN_ON(!genpd->opp_table); + WARN_ON(IS_ERR(genpd->opp_table)); } genpd->provider = &np->fwnode; diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 9d7fb45b1786..9f1676989483 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -1063,7 +1063,7 @@ static struct opp_table *_allocate_opp_table(struct device *dev, int index) */ opp_table = kzalloc(sizeof(*opp_table), GFP_KERNEL); if (!opp_table) - return NULL; + return ERR_PTR(-ENOMEM); mutex_init(&opp_table->lock); mutex_init(&opp_table->genpd_virt_dev_lock); @@ -1074,8 +1074,8 @@ static struct opp_table *_allocate_opp_table(struct device *dev, int index) opp_dev = _add_opp_dev(dev, opp_table); if (!opp_dev) { - kfree(opp_table); - return NULL; + ret = -ENOMEM; + goto err; } _of_init_opp_table(opp_table, dev, index); @@ -1084,16 +1084,21 @@ static struct opp_table *_allocate_opp_table(struct device *dev, int index) opp_table->clk = clk_get(dev, NULL); if (IS_ERR(opp_table->clk)) { ret = PTR_ERR(opp_table->clk); - if (ret != -EPROBE_DEFER) - dev_dbg(dev, "%s: Couldn't find clock: %d\n", __func__, - ret); + if (ret == -EPROBE_DEFER) + goto err; + + dev_dbg(dev, "%s: Couldn't find clock: %d\n", __func__, ret); } /* Find interconnect path(s) for the device */ ret = dev_pm_opp_of_find_icc_paths(dev, opp_table); - if (ret) + if (ret) { + if (ret == -EPROBE_DEFER) + goto err; + dev_warn(dev, "%s: Error finding interconnect paths: %d\n", __func__, ret); + } BLOCKING_INIT_NOTIFIER_HEAD(&opp_table->head); INIT_LIST_HEAD(&opp_table->opp_list); @@ -1102,6 +1107,10 @@ static struct opp_table *_allocate_opp_table(struct device *dev, int index) /* Secure the device table modification */ list_add(&opp_table->node, &opp_tables); return opp_table; + +err: + kfree(opp_table); + return ERR_PTR(ret); } void _get_opp_table_kref(struct opp_table *opp_table) @@ -1124,7 +1133,7 @@ static struct opp_table *_opp_get_opp_table(struct device *dev, int index) if (opp_table) { if (!_add_opp_dev_unlocked(dev, opp_table)) { dev_pm_opp_put_opp_table(opp_table); - opp_table = NULL; + opp_table = ERR_PTR(-ENOMEM); } goto unlock; } @@ -1568,8 +1577,8 @@ struct opp_table *dev_pm_opp_set_supported_hw(struct device *dev, struct opp_table *opp_table; opp_table = dev_pm_opp_get_opp_table(dev); - if (!opp_table) - return ERR_PTR(-ENOMEM); + if (IS_ERR(opp_table)) + return opp_table; /* Make sure there are no concurrent readers while updating opp_table */ WARN_ON(!list_empty(&opp_table->opp_list)); @@ -1627,8 +1636,8 @@ struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name) struct opp_table *opp_table; opp_table = dev_pm_opp_get_opp_table(dev); - if (!opp_table) - return ERR_PTR(-ENOMEM); + if (IS_ERR(opp_table)) + return opp_table; /* Make sure there are no concurrent readers while updating opp_table */ WARN_ON(!list_empty(&opp_table->opp_list)); @@ -1720,8 +1729,8 @@ struct opp_table *dev_pm_opp_set_regulators(struct device *dev, int ret, i; opp_table = dev_pm_opp_get_opp_table(dev); - if (!opp_table) - return ERR_PTR(-ENOMEM); + if (IS_ERR(opp_table)) + return opp_table; /* This should be called before OPPs are initialized */ if (WARN_ON(!list_empty(&opp_table->opp_list))) { @@ -1830,8 +1839,8 @@ struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char *name) int ret; opp_table = dev_pm_opp_get_opp_table(dev); - if (!opp_table) - return ERR_PTR(-ENOMEM); + if (IS_ERR(opp_table)) + return opp_table; /* This should be called before OPPs are initialized */ if (WARN_ON(!list_empty(&opp_table->opp_list))) { @@ -1898,8 +1907,8 @@ struct opp_table *dev_pm_opp_register_set_opp_helper(struct device *dev, return ERR_PTR(-EINVAL); opp_table = dev_pm_opp_get_opp_table(dev); - if (!opp_table) - return ERR_PTR(-ENOMEM); + if (!IS_ERR(opp_table)) + return opp_table; /* This should be called before OPPs are initialized */ if (WARN_ON(!list_empty(&opp_table->opp_list))) { @@ -1979,8 +1988,8 @@ struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, const char **name = names; opp_table = dev_pm_opp_get_opp_table(dev); - if (!opp_table) - return ERR_PTR(-ENOMEM); + if (IS_ERR(opp_table)) + return opp_table; /* * If the genpd's OPP table isn't already initialized, parsing of the @@ -2150,8 +2159,8 @@ int dev_pm_opp_add(struct device *dev, unsigned long freq, unsigned long u_volt) int ret; opp_table = dev_pm_opp_get_opp_table(dev); - if (!opp_table) - return -ENOMEM; + if (IS_ERR(opp_table)) + return PTR_ERR(opp_table); /* Fix regulator count for dynamic OPPs */ opp_table->regulator_count = 1; diff --git a/drivers/opp/of.c b/drivers/opp/of.c index 0430290670ab..d8b623cc015a 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -947,8 +947,8 @@ int dev_pm_opp_of_add_table(struct device *dev) int ret; opp_table = dev_pm_opp_get_opp_table_indexed(dev, 0); - if (!opp_table) - return -ENOMEM; + if (IS_ERR(opp_table)) + return PTR_ERR(opp_table); /* * OPPs have two version of bindings now. Also try the old (v1) @@ -1002,8 +1002,8 @@ int dev_pm_opp_of_add_table_indexed(struct device *dev, int index) } opp_table = dev_pm_opp_get_opp_table_indexed(dev, index); - if (!opp_table) - return -ENOMEM; + if (IS_ERR(opp_table)) + return PTR_ERR(opp_table); ret = _of_add_opp_table_v2(dev, opp_table); if (ret) diff --git a/drivers/soc/samsung/exynos-asv.c b/drivers/soc/samsung/exynos-asv.c index 30bb7b7cc769..8abf4dfaa5c5 100644 --- a/drivers/soc/samsung/exynos-asv.c +++ b/drivers/soc/samsung/exynos-asv.c @@ -93,7 +93,7 @@ static int exynos_asv_update_opps(struct exynos_asv *asv) continue; opp_table = dev_pm_opp_get_opp_table(cpu); - if (IS_ERR_OR_NULL(opp_table)) + if (IS_ERR(opp_table)) continue; if (!last_opp_table || opp_table != last_opp_table) { From 6545eb030e6f14cef8793a86312483c788eaee46 Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Tue, 25 Aug 2020 13:47:39 +0100 Subject: [PATCH 007/265] objtool: Move object file loading out of check() Structure objtool_file can be used by different subcommands. In fact it already is, by check and orc. Provide a function that allows to initialize objtool_file, that builtin can call, without relying on check to do the correct setup for them and explicitly hand the objtool_file to them. Reviewed-by: Miroslav Benes Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- tools/objtool/builtin-check.c | 7 +++++- tools/objtool/builtin-orc.c | 8 ++++++- tools/objtool/check.c | 42 +++++++++++------------------------ tools/objtool/objtool.c | 30 +++++++++++++++++++++++++ tools/objtool/objtool.h | 4 +++- tools/objtool/weak.c | 4 +--- 6 files changed, 60 insertions(+), 35 deletions(-) diff --git a/tools/objtool/builtin-check.c b/tools/objtool/builtin-check.c index 7a44174967b5..0126ec3bb6c9 100644 --- a/tools/objtool/builtin-check.c +++ b/tools/objtool/builtin-check.c @@ -41,6 +41,7 @@ const struct option check_options[] = { int cmd_check(int argc, const char **argv) { const char *objname, *s; + struct objtool_file *file; argc = parse_options(argc, argv, check_options, check_usage, 0); @@ -53,5 +54,9 @@ int cmd_check(int argc, const char **argv) if (s && !s[9]) vmlinux = true; - return check(objname, false); + file = objtool_open_read(objname); + if (!file) + return 1; + + return check(file, false); } diff --git a/tools/objtool/builtin-orc.c b/tools/objtool/builtin-orc.c index b1dfe2007962..3979f275a775 100644 --- a/tools/objtool/builtin-orc.c +++ b/tools/objtool/builtin-orc.c @@ -31,13 +31,19 @@ int cmd_orc(int argc, const char **argv) usage_with_options(orc_usage, check_options); if (!strncmp(argv[0], "gen", 3)) { + struct objtool_file *file; + argc = parse_options(argc, argv, check_options, orc_usage, 0); if (argc != 1) usage_with_options(orc_usage, check_options); objname = argv[0]; - return check(objname, true); + file = objtool_open_read(objname); + if (!file) + return 1; + + return check(file, true); } if (!strcmp(argv[0], "dump")) { diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 75d0cd2f9044..9d4efa3b12ba 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -28,7 +28,6 @@ struct alternative { bool skip_orig; }; -const char *objname; struct cfi_init_state initial_func_cfi; struct instruction *find_insn(struct objtool_file *file, @@ -2909,37 +2908,22 @@ static int validate_reachable_instructions(struct objtool_file *file) return 0; } -static struct objtool_file file; - -int check(const char *_objname, bool orc) +int check(struct objtool_file *file, bool orc) { int ret, warnings = 0; - objname = _objname; - - file.elf = elf_open_read(objname, O_RDWR); - if (!file.elf) - return 1; - - INIT_LIST_HEAD(&file.insn_list); - hash_init(file.insn_hash); - INIT_LIST_HEAD(&file.static_call_list); - file.c_file = !vmlinux && find_section_by_name(file.elf, ".comment"); - file.ignore_unreachables = no_unreachable; - file.hints = false; - arch_initial_func_cfi_state(&initial_func_cfi); - ret = decode_sections(&file); + ret = decode_sections(file); if (ret < 0) goto out; warnings += ret; - if (list_empty(&file.insn_list)) + if (list_empty(&file->insn_list)) goto out; if (vmlinux && !validate_dup) { - ret = validate_vmlinux_functions(&file); + ret = validate_vmlinux_functions(file); if (ret < 0) goto out; @@ -2948,46 +2932,46 @@ int check(const char *_objname, bool orc) } if (retpoline) { - ret = validate_retpoline(&file); + ret = validate_retpoline(file); if (ret < 0) return ret; warnings += ret; } - ret = validate_functions(&file); + ret = validate_functions(file); if (ret < 0) goto out; warnings += ret; - ret = validate_unwind_hints(&file, NULL); + ret = validate_unwind_hints(file, NULL); if (ret < 0) goto out; warnings += ret; if (!warnings) { - ret = validate_reachable_instructions(&file); + ret = validate_reachable_instructions(file); if (ret < 0) goto out; warnings += ret; } - ret = create_static_call_sections(&file); + ret = create_static_call_sections(file); if (ret < 0) goto out; warnings += ret; if (orc) { - ret = create_orc(&file); + ret = create_orc(file); if (ret < 0) goto out; - ret = create_orc_sections(&file); + ret = create_orc_sections(file); if (ret < 0) goto out; } - if (file.elf->changed) { - ret = elf_write(file.elf); + if (file->elf->changed) { + ret = elf_write(file->elf); if (ret < 0) goto out; } diff --git a/tools/objtool/objtool.c b/tools/objtool/objtool.c index 58fdda510653..9df0cd86d310 100644 --- a/tools/objtool/objtool.c +++ b/tools/objtool/objtool.c @@ -22,6 +22,8 @@ #include #include "builtin.h" +#include "objtool.h" +#include "warn.h" struct cmd_struct { const char *name; @@ -39,6 +41,34 @@ static struct cmd_struct objtool_cmds[] = { bool help; +const char *objname; +static struct objtool_file file; + +struct objtool_file *objtool_open_read(const char *_objname) +{ + if (objname) { + if (strcmp(objname, _objname)) { + WARN("won't handle more than one file at a time"); + return NULL; + } + return &file; + } + objname = _objname; + + file.elf = elf_open_read(objname, O_RDWR); + if (!file.elf) + return NULL; + + INIT_LIST_HEAD(&file.insn_list); + hash_init(file.insn_hash); + INIT_LIST_HEAD(&file.static_call_list); + file.c_file = !vmlinux && find_section_by_name(file.elf, ".comment"); + file.ignore_unreachables = no_unreachable; + file.hints = false; + + return &file; +} + static void cmd_usage(void) { unsigned int i, longest = 0; diff --git a/tools/objtool/objtool.h b/tools/objtool/objtool.h index 9a7cd0b88bd8..7efc43f22174 100644 --- a/tools/objtool/objtool.h +++ b/tools/objtool/objtool.h @@ -20,7 +20,9 @@ struct objtool_file { bool ignore_unreachables, c_file, hints, rodata; }; -int check(const char *objname, bool orc); +struct objtool_file *objtool_open_read(const char *_objname); + +int check(struct objtool_file *file, bool orc); int orc_dump(const char *objname); int create_orc(struct objtool_file *file); int create_orc_sections(struct objtool_file *file); diff --git a/tools/objtool/weak.c b/tools/objtool/weak.c index 942ea5e8ac36..82698319f008 100644 --- a/tools/objtool/weak.c +++ b/tools/objtool/weak.c @@ -17,9 +17,7 @@ return ENOSYS; \ }) -const char __weak *objname; - -int __weak check(const char *_objname, bool orc) +int __weak check(struct objtool_file *file, bool orc) { UNSUPPORTED("check subcommand"); } From d44becb9decf4438d1e555b1428634964d2e5764 Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Tue, 25 Aug 2020 13:47:40 +0100 Subject: [PATCH 008/265] objtool: Move ORC logic out of check() Now that the objtool_file can be obtained outside of the check function, orc generation builtin no longer requires check to explicitly call its orc related functions. Signed-off-by: Julien Thierry Reviewed-by: Miroslav Benes Signed-off-by: Josh Poimboeuf --- tools/objtool/builtin-check.c | 10 +++++++++- tools/objtool/builtin-orc.c | 21 ++++++++++++++++++++- tools/objtool/check.c | 18 +----------------- tools/objtool/objtool.h | 2 +- tools/objtool/weak.c | 2 +- 5 files changed, 32 insertions(+), 21 deletions(-) diff --git a/tools/objtool/builtin-check.c b/tools/objtool/builtin-check.c index 0126ec3bb6c9..c6d199bfd0ae 100644 --- a/tools/objtool/builtin-check.c +++ b/tools/objtool/builtin-check.c @@ -42,6 +42,7 @@ int cmd_check(int argc, const char **argv) { const char *objname, *s; struct objtool_file *file; + int ret; argc = parse_options(argc, argv, check_options, check_usage, 0); @@ -58,5 +59,12 @@ int cmd_check(int argc, const char **argv) if (!file) return 1; - return check(file, false); + ret = check(file); + if (ret) + return ret; + + if (file->elf->changed) + return elf_write(file->elf); + + return 0; } diff --git a/tools/objtool/builtin-orc.c b/tools/objtool/builtin-orc.c index 3979f275a775..7b31121fa60b 100644 --- a/tools/objtool/builtin-orc.c +++ b/tools/objtool/builtin-orc.c @@ -32,6 +32,7 @@ int cmd_orc(int argc, const char **argv) if (!strncmp(argv[0], "gen", 3)) { struct objtool_file *file; + int ret; argc = parse_options(argc, argv, check_options, orc_usage, 0); if (argc != 1) @@ -43,7 +44,25 @@ int cmd_orc(int argc, const char **argv) if (!file) return 1; - return check(file, true); + ret = check(file); + if (ret) + return ret; + + if (list_empty(&file->insn_list)) + return 0; + + ret = create_orc(file); + if (ret) + return ret; + + ret = create_orc_sections(file); + if (ret) + return ret; + + if (!file->elf->changed) + return 0; + + return elf_write(file->elf); } if (!strcmp(argv[0], "dump")) { diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 9d4efa3b12ba..4afc2d5465b9 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2908,7 +2908,7 @@ static int validate_reachable_instructions(struct objtool_file *file) return 0; } -int check(struct objtool_file *file, bool orc) +int check(struct objtool_file *file) { int ret, warnings = 0; @@ -2960,22 +2960,6 @@ int check(struct objtool_file *file, bool orc) goto out; warnings += ret; - if (orc) { - ret = create_orc(file); - if (ret < 0) - goto out; - - ret = create_orc_sections(file); - if (ret < 0) - goto out; - } - - if (file->elf->changed) { - ret = elf_write(file->elf); - if (ret < 0) - goto out; - } - out: if (ret < 0) { /* diff --git a/tools/objtool/objtool.h b/tools/objtool/objtool.h index 7efc43f22174..a635f68a9b09 100644 --- a/tools/objtool/objtool.h +++ b/tools/objtool/objtool.h @@ -22,7 +22,7 @@ struct objtool_file { struct objtool_file *objtool_open_read(const char *_objname); -int check(struct objtool_file *file, bool orc); +int check(struct objtool_file *file); int orc_dump(const char *objname); int create_orc(struct objtool_file *file); int create_orc_sections(struct objtool_file *file); diff --git a/tools/objtool/weak.c b/tools/objtool/weak.c index 82698319f008..29180d599b08 100644 --- a/tools/objtool/weak.c +++ b/tools/objtool/weak.c @@ -17,7 +17,7 @@ return ENOSYS; \ }) -int __weak check(struct objtool_file *file, bool orc) +int __weak check(struct objtool_file *file) { UNSUPPORTED("check subcommand"); } From 3eaecac88a17f7fdf29561a197dc728f7f697c60 Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Tue, 25 Aug 2020 13:47:41 +0100 Subject: [PATCH 009/265] objtool: Skip ORC entry creation for non-text sections Orc generation is only done for text sections, but some instructions can be found in non-text sections (e.g. .discard.text sections). Skip setting their orc sections since their whole sections will be skipped for orc generation. Reviewed-by: Miroslav Benes Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- tools/objtool/orc_gen.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c index e6b2363c2e03..22fe4398197f 100644 --- a/tools/objtool/orc_gen.c +++ b/tools/objtool/orc_gen.c @@ -18,6 +18,9 @@ int create_orc(struct objtool_file *file) struct cfi_reg *cfa = &insn->cfi.cfa; struct cfi_reg *bp = &insn->cfi.regs[CFI_BP]; + if (!insn->sec->text) + continue; + orc->end = insn->cfi.end; if (cfa->base == CFI_UNDEFINED) { From 66734e32463bd1346466f92662feeaccef26e94f Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Tue, 25 Aug 2020 13:47:42 +0100 Subject: [PATCH 010/265] objtool: Define 'struct orc_entry' only when needed Implementation of ORC requires some definitions that are currently provided by the target architecture headers. Do not depend on these definitions when the orc subcommand is not implemented. This avoid requiring arches with no orc implementation to provide dummy orc definitions. Signed-off-by: Julien Thierry Reviewed-by: Miroslav Benes Signed-off-by: Josh Poimboeuf --- tools/objtool/Makefile | 4 ++++ tools/objtool/arch.h | 2 ++ tools/objtool/check.h | 2 ++ 3 files changed, 8 insertions(+) diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile index 7770edcda3a0..33d1e3ca8efd 100644 --- a/tools/objtool/Makefile +++ b/tools/objtool/Makefile @@ -55,6 +55,10 @@ ifeq ($(SRCARCH),x86) SUBCMD_ORC := y endif +ifeq ($(SUBCMD_ORC),y) + CFLAGS += -DINSN_USE_ORC +endif + export SUBCMD_CHECK SUBCMD_ORC export srctree OUTPUT CFLAGS SRCARCH AWK include $(srctree)/tools/build/Makefile.include diff --git a/tools/objtool/arch.h b/tools/objtool/arch.h index 2e2ce089b0e9..b18c5f61d42d 100644 --- a/tools/objtool/arch.h +++ b/tools/objtool/arch.h @@ -11,7 +11,9 @@ #include "objtool.h" #include "cfi.h" +#ifdef INSN_USE_ORC #include +#endif enum insn_type { INSN_JUMP_CONDITIONAL, diff --git a/tools/objtool/check.h b/tools/objtool/check.h index 36d38b9153ac..6cac34542122 100644 --- a/tools/objtool/check.h +++ b/tools/objtool/check.h @@ -43,7 +43,9 @@ struct instruction { struct symbol *func; struct list_head stack_ops; struct cfi_state cfi; +#ifdef INSN_USE_ORC struct orc_entry orc; +#endif }; struct instruction *find_insn(struct objtool_file *file, From 84b43284af40742abeb2cdd6998a4084866ba015 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 4 Sep 2020 18:27:27 +0200 Subject: [PATCH 011/265] ACPICA: Validate GPE blocks at init time Some of the checks done by acpi_hw_read() and acpi_hw_write(), which are used for accessing GPE registers, are redundant in the specific case of GPE registers and the ones that are not redundant can be done upfront at the initialization time so as to fail the initialization if they are not passed instead of failing every access to the affected GPE registers going forward (including accesses from the SCI interrupt handler). Modify the GPE blocks initialization code accordingly. Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpica/achware.h | 2 ++ drivers/acpi/acpica/evgpeblk.c | 17 +++++++++++++++++ drivers/acpi/acpica/hwvalid.c | 30 ++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/drivers/acpi/acpica/achware.h b/drivers/acpi/acpica/achware.h index ebf6453d0e21..f1f644b58b15 100644 --- a/drivers/acpi/acpica/achware.h +++ b/drivers/acpi/acpica/achware.h @@ -73,6 +73,8 @@ acpi_status acpi_hw_read_port(acpi_io_address address, u32 *value, u32 width); acpi_status acpi_hw_write_port(acpi_io_address address, u32 value, u32 width); +acpi_status acpi_hw_validate_io_block(u64 address, u32 bit_width, u32 count); + /* * hwgpe - GPE support */ diff --git a/drivers/acpi/acpica/evgpeblk.c b/drivers/acpi/acpica/evgpeblk.c index 132adff1e131..eb5d98757fdc 100644 --- a/drivers/acpi/acpica/evgpeblk.c +++ b/drivers/acpi/acpica/evgpeblk.c @@ -317,6 +317,23 @@ acpi_ev_create_gpe_block(struct acpi_namespace_node *gpe_device, return_ACPI_STATUS(AE_OK); } + /* Validate the space_ID */ + + if ((space_id != ACPI_ADR_SPACE_SYSTEM_MEMORY) && + (space_id != ACPI_ADR_SPACE_SYSTEM_IO)) { + ACPI_ERROR((AE_INFO, + "Unsupported address space: 0x%X", space_id)); + return_ACPI_STATUS(AE_SUPPORT); + } + + if (space_id == ACPI_ADR_SPACE_SYSTEM_IO) { + status = acpi_hw_validate_io_block(address, + ACPI_GPE_REGISTER_WIDTH, + register_count); + if (ACPI_FAILURE(status)) + return_ACPI_STATUS(status); + } + /* Allocate a new GPE block */ gpe_block = ACPI_ALLOCATE_ZEROED(sizeof(struct acpi_gpe_block_info)); diff --git a/drivers/acpi/acpica/hwvalid.c b/drivers/acpi/acpica/hwvalid.c index 4d94861e6093..b2ca7dfd3fc9 100644 --- a/drivers/acpi/acpica/hwvalid.c +++ b/drivers/acpi/acpica/hwvalid.c @@ -292,3 +292,33 @@ acpi_status acpi_hw_write_port(acpi_io_address address, u32 value, u32 width) return (AE_OK); } + +/****************************************************************************** + * + * FUNCTION: acpi_hw_validate_io_block + * + * PARAMETERS: Address Address of I/O port/register blobk + * bit_width Number of bits (8,16,32) in each register + * count Number of registers in the block + * + * RETURN: Status + * + * DESCRIPTION: Validates a block of I/O ports/registers. + * + ******************************************************************************/ + +acpi_status acpi_hw_validate_io_block(u64 address, u32 bit_width, u32 count) +{ + acpi_status status; + + while (count--) { + status = acpi_hw_validate_io_request((acpi_io_address)address, + bit_width); + if (ACPI_FAILURE(status)) + return_ACPI_STATUS(status); + + address += ACPI_DIV_8(bit_width); + } + + return_ACPI_STATUS(AE_OK); +} From f06011ad6225b0f5e4246b5635c5570fa8d5fcdf Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 4 Sep 2020 18:27:43 +0200 Subject: [PATCH 012/265] ACPICA: Introduce acpi_hw_gpe_read() and acpi_hw_gpe_write() Now that GPE blocks are validated at the initialization time, accesses to GPE registers can be made more straightforward by ommitting all of the redundant checks in acpi_hw_read() and acpi_hw_write() and only invoking the OS-provided helper for the given type of access (read or write) and the address space holding these registers. For this reason, introduce simplified routines for accessing GPE registers, acpi_hw_gpe_read() and acpi_hw_gpe_write(), designed in accordance with the above observation, and modify all of the code accessing GPE registers to use them instead of acpi_hw_read() and acpi_hw_write(), respectively. Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpica/achware.h | 4 ++ drivers/acpi/acpica/evgpe.c | 4 +- drivers/acpi/acpica/evgpeblk.c | 4 +- drivers/acpi/acpica/hwgpe.c | 92 ++++++++++++++++++++++++++++------ 4 files changed, 84 insertions(+), 20 deletions(-) diff --git a/drivers/acpi/acpica/achware.h b/drivers/acpi/acpica/achware.h index f1f644b58b15..4dba7229f9c1 100644 --- a/drivers/acpi/acpica/achware.h +++ b/drivers/acpi/acpica/achware.h @@ -78,6 +78,10 @@ acpi_status acpi_hw_validate_io_block(u64 address, u32 bit_width, u32 count); /* * hwgpe - GPE support */ +acpi_status acpi_hw_gpe_read(u64 *value, struct acpi_generic_address *reg); + +acpi_status acpi_hw_gpe_write(u64 value, struct acpi_generic_address *reg); + u32 acpi_hw_get_gpe_register_bit(struct acpi_gpe_event_info *gpe_event_info); acpi_status diff --git a/drivers/acpi/acpica/evgpe.c b/drivers/acpi/acpica/evgpe.c index 3e39907fedd9..06b9c8dd11c9 100644 --- a/drivers/acpi/acpica/evgpe.c +++ b/drivers/acpi/acpica/evgpe.c @@ -656,14 +656,14 @@ acpi_ev_detect_gpe(struct acpi_namespace_node *gpe_device, /* GPE currently enabled (enable bit == 1)? */ - status = acpi_hw_read(&enable_reg, &gpe_register_info->enable_address); + status = acpi_hw_gpe_read(&enable_reg, &gpe_register_info->enable_address); if (ACPI_FAILURE(status)) { goto error_exit; } /* GPE currently active (status bit == 1)? */ - status = acpi_hw_read(&status_reg, &gpe_register_info->status_address); + status = acpi_hw_gpe_read(&status_reg, &gpe_register_info->status_address); if (ACPI_FAILURE(status)) { goto error_exit; } diff --git a/drivers/acpi/acpica/evgpeblk.c b/drivers/acpi/acpica/evgpeblk.c index eb5d98757fdc..150c916dca5e 100644 --- a/drivers/acpi/acpica/evgpeblk.c +++ b/drivers/acpi/acpica/evgpeblk.c @@ -251,14 +251,14 @@ acpi_ev_create_gpe_info_blocks(struct acpi_gpe_block_info *gpe_block) /* Disable all GPEs within this register */ - status = acpi_hw_write(0x00, &this_register->enable_address); + status = acpi_hw_gpe_write(0x00, &this_register->enable_address); if (ACPI_FAILURE(status)) { goto error_exit; } /* Clear any pending GPE events within this register */ - status = acpi_hw_write(0xFF, &this_register->status_address); + status = acpi_hw_gpe_write(0xFF, &this_register->status_address); if (ACPI_FAILURE(status)) { goto error_exit; } diff --git a/drivers/acpi/acpica/hwgpe.c b/drivers/acpi/acpica/hwgpe.c index 49c46d4dd070..6cc88524839d 100644 --- a/drivers/acpi/acpica/hwgpe.c +++ b/drivers/acpi/acpica/hwgpe.c @@ -24,6 +24,66 @@ static acpi_status acpi_hw_gpe_enable_write(u8 enable_mask, struct acpi_gpe_register_info *gpe_register_info); +/****************************************************************************** + * + * FUNCTION: acpi_hw_gpe_read + * + * PARAMETERS: value - Where the value is returned + * reg - GAS register structure + * + * RETURN: Status + * + * DESCRIPTION: Read from a GPE register in either memory or IO space. + * + * LIMITATIONS: + * space_ID must be system_memory or system_IO. + * + ******************************************************************************/ + +acpi_status acpi_hw_gpe_read(u64 *value, struct acpi_generic_address *reg) +{ + acpi_status status; + u32 value32; + + if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) { + return acpi_os_read_memory((acpi_physical_address)reg->address, + value, ACPI_GPE_REGISTER_WIDTH); + } + + status = acpi_os_read_port((acpi_io_address)reg->address, + &value32, ACPI_GPE_REGISTER_WIDTH); + if (ACPI_FAILURE(status)) + return_ACPI_STATUS(status); + + *value = (u64)value32; + + return_ACPI_STATUS(AE_OK); +} + +/****************************************************************************** + * + * FUNCTION: acpi_hw_gpe_write + * + * PARAMETERS: value - Value to be written + * reg - GAS register structure + * + * RETURN: Status + * + * DESCRIPTION: Write to a GPE register in either memory or IO space. + * + ******************************************************************************/ + +acpi_status acpi_hw_gpe_write(u64 value, struct acpi_generic_address *reg) +{ + if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) { + return acpi_os_write_memory((acpi_physical_address)reg->address, + value, ACPI_GPE_REGISTER_WIDTH); + } + + return acpi_os_write_port((acpi_io_address)reg->address, (u32)value, + ACPI_GPE_REGISTER_WIDTH); +} + /****************************************************************************** * * FUNCTION: acpi_hw_get_gpe_register_bit @@ -79,7 +139,8 @@ acpi_hw_low_set_gpe(struct acpi_gpe_event_info *gpe_event_info, u32 action) /* Get current value of the enable register that contains this GPE */ - status = acpi_hw_read(&enable_mask, &gpe_register_info->enable_address); + status = acpi_hw_gpe_read(&enable_mask, + &gpe_register_info->enable_address); if (ACPI_FAILURE(status)) { return (status); } @@ -118,9 +179,8 @@ acpi_hw_low_set_gpe(struct acpi_gpe_event_info *gpe_event_info, u32 action) /* Write the updated enable mask */ - status = - acpi_hw_write(enable_mask, - &gpe_register_info->enable_address); + status = acpi_hw_gpe_write(enable_mask, + &gpe_register_info->enable_address); } return (status); } @@ -158,8 +218,8 @@ acpi_status acpi_hw_clear_gpe(struct acpi_gpe_event_info *gpe_event_info) */ register_bit = acpi_hw_get_gpe_register_bit(gpe_event_info); - status = - acpi_hw_write(register_bit, &gpe_register_info->status_address); + status = acpi_hw_gpe_write(register_bit, + &gpe_register_info->status_address); return (status); } @@ -227,7 +287,7 @@ acpi_hw_get_gpe_status(struct acpi_gpe_event_info *gpe_event_info, /* GPE currently enabled (enable bit == 1)? */ - status = acpi_hw_read(&in_byte, &gpe_register_info->enable_address); + status = acpi_hw_gpe_read(&in_byte, &gpe_register_info->enable_address); if (ACPI_FAILURE(status)) { return (status); } @@ -238,7 +298,7 @@ acpi_hw_get_gpe_status(struct acpi_gpe_event_info *gpe_event_info, /* GPE currently active (status bit == 1)? */ - status = acpi_hw_read(&in_byte, &gpe_register_info->status_address); + status = acpi_hw_gpe_read(&in_byte, &gpe_register_info->status_address); if (ACPI_FAILURE(status)) { return (status); } @@ -274,7 +334,8 @@ acpi_hw_gpe_enable_write(u8 enable_mask, gpe_register_info->enable_mask = enable_mask; - status = acpi_hw_write(enable_mask, &gpe_register_info->enable_address); + status = acpi_hw_gpe_write(enable_mask, + &gpe_register_info->enable_address); return (status); } @@ -341,9 +402,8 @@ acpi_hw_clear_gpe_block(struct acpi_gpe_xrupt_info *gpe_xrupt_info, /* Clear status on all GPEs in this register */ - status = - acpi_hw_write(0xFF, - &gpe_block->register_info[i].status_address); + status = acpi_hw_gpe_write(0xFF, + &gpe_block->register_info[i].status_address); if (ACPI_FAILURE(status)) { return (status); } @@ -481,14 +541,14 @@ acpi_hw_get_gpe_block_status(struct acpi_gpe_xrupt_info *gpe_xrupt_info, for (i = 0; i < gpe_block->register_count; i++) { gpe_register_info = &gpe_block->register_info[i]; - status = acpi_hw_read(&in_enable, - &gpe_register_info->enable_address); + status = acpi_hw_gpe_read(&in_enable, + &gpe_register_info->enable_address); if (ACPI_FAILURE(status)) { continue; } - status = acpi_hw_read(&in_status, - &gpe_register_info->status_address); + status = acpi_hw_gpe_read(&in_status, + &gpe_register_info->status_address); if (ACPI_FAILURE(status)) { continue; } From 9da8e9ac171438525e4c1c377609818ef1a6237b Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 4 Sep 2020 18:27:52 +0200 Subject: [PATCH 013/265] ACPICA: Introduce special struct type for GPE register addresses Notice that the bit_width, bit_offset and access_width fields in struct acpi_generic_address are not used during GPE register accesses any more, so introduce a simplified address structure type, struct acpi_gpe_address, to represent addresses of GPE registers and use it instead of struct acpi_generic_address in struct acpi_gpe_register_info. Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpica/achware.h | 4 ++-- drivers/acpi/acpica/aclocal.h | 11 +++++++++-- drivers/acpi/acpica/evgpeblk.c | 6 ------ drivers/acpi/acpica/hwgpe.c | 8 ++++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/drivers/acpi/acpica/achware.h b/drivers/acpi/acpica/achware.h index 4dba7229f9c1..6ab92e28330d 100644 --- a/drivers/acpi/acpica/achware.h +++ b/drivers/acpi/acpica/achware.h @@ -78,9 +78,9 @@ acpi_status acpi_hw_validate_io_block(u64 address, u32 bit_width, u32 count); /* * hwgpe - GPE support */ -acpi_status acpi_hw_gpe_read(u64 *value, struct acpi_generic_address *reg); +acpi_status acpi_hw_gpe_read(u64 *value, struct acpi_gpe_address *reg); -acpi_status acpi_hw_gpe_write(u64 value, struct acpi_generic_address *reg); +acpi_status acpi_hw_gpe_write(u64 value, struct acpi_gpe_address *reg); u32 acpi_hw_get_gpe_register_bit(struct acpi_gpe_event_info *gpe_event_info); diff --git a/drivers/acpi/acpica/aclocal.h b/drivers/acpi/acpica/aclocal.h index af58cd2dc9d3..f83b98fa13ac 100644 --- a/drivers/acpi/acpica/aclocal.h +++ b/drivers/acpi/acpica/aclocal.h @@ -454,11 +454,18 @@ struct acpi_gpe_event_info { u8 disable_for_dispatch; /* Masked during dispatching */ }; +/* GPE register address */ + +struct acpi_gpe_address { + u8 space_id; /* Address space where the register exists */ + u64 address; /* 64-bit address of the register */ +}; + /* Information about a GPE register pair, one per each status/enable pair in an array */ struct acpi_gpe_register_info { - struct acpi_generic_address status_address; /* Address of status reg */ - struct acpi_generic_address enable_address; /* Address of enable reg */ + struct acpi_gpe_address status_address; /* Address of status reg */ + struct acpi_gpe_address enable_address; /* Address of enable reg */ u16 base_gpe_number; /* Base GPE number for this register */ u8 enable_for_wake; /* GPEs to keep enabled when sleeping */ u8 enable_for_run; /* GPEs to keep enabled when running */ diff --git a/drivers/acpi/acpica/evgpeblk.c b/drivers/acpi/acpica/evgpeblk.c index 150c916dca5e..f5298be4273a 100644 --- a/drivers/acpi/acpica/evgpeblk.c +++ b/drivers/acpi/acpica/evgpeblk.c @@ -233,12 +233,6 @@ acpi_ev_create_gpe_info_blocks(struct acpi_gpe_block_info *gpe_block) this_register->status_address.space_id = gpe_block->space_id; this_register->enable_address.space_id = gpe_block->space_id; - this_register->status_address.bit_width = - ACPI_GPE_REGISTER_WIDTH; - this_register->enable_address.bit_width = - ACPI_GPE_REGISTER_WIDTH; - this_register->status_address.bit_offset = 0; - this_register->enable_address.bit_offset = 0; /* Init the event_info for each GPE within this register */ diff --git a/drivers/acpi/acpica/hwgpe.c b/drivers/acpi/acpica/hwgpe.c index 6cc88524839d..a0e71f34c77a 100644 --- a/drivers/acpi/acpica/hwgpe.c +++ b/drivers/acpi/acpica/hwgpe.c @@ -29,7 +29,7 @@ acpi_hw_gpe_enable_write(u8 enable_mask, * FUNCTION: acpi_hw_gpe_read * * PARAMETERS: value - Where the value is returned - * reg - GAS register structure + * reg - GPE register structure * * RETURN: Status * @@ -40,7 +40,7 @@ acpi_hw_gpe_enable_write(u8 enable_mask, * ******************************************************************************/ -acpi_status acpi_hw_gpe_read(u64 *value, struct acpi_generic_address *reg) +acpi_status acpi_hw_gpe_read(u64 *value, struct acpi_gpe_address *reg) { acpi_status status; u32 value32; @@ -65,7 +65,7 @@ acpi_status acpi_hw_gpe_read(u64 *value, struct acpi_generic_address *reg) * FUNCTION: acpi_hw_gpe_write * * PARAMETERS: value - Value to be written - * reg - GAS register structure + * reg - GPE register structure * * RETURN: Status * @@ -73,7 +73,7 @@ acpi_status acpi_hw_gpe_read(u64 *value, struct acpi_generic_address *reg) * ******************************************************************************/ -acpi_status acpi_hw_gpe_write(u64 value, struct acpi_generic_address *reg) +acpi_status acpi_hw_gpe_write(u64 value, struct acpi_gpe_address *reg) { if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) { return acpi_os_write_memory((acpi_physical_address)reg->address, From 6ccbd29ade0d159ee1be398dc9defaae567c253d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:02 +0200 Subject: [PATCH 014/265] KVM: SVM: nested: Don't allocate VMCB structures on stack Do not allocate a vmcb_control_area and a vmcb_save_area on the stack, as these structures will become larger with future extenstions of SVM and thus the svm_set_nested_state() function will become a too large stack frame. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-2-joro@8bytes.org --- arch/x86/kvm/svm/nested.c | 47 +++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index fb68467e6049..28036629abf8 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -1060,10 +1060,14 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, struct vmcb *hsave = svm->nested.hsave; struct vmcb __user *user_vmcb = (struct vmcb __user *) &user_kvm_nested_state->data.svm[0]; - struct vmcb_control_area ctl; - struct vmcb_save_area save; + struct vmcb_control_area *ctl; + struct vmcb_save_area *save; + int ret; u32 cr0; + BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) > + KVM_STATE_NESTED_SVM_VMCB_SIZE); + if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM) return -EINVAL; @@ -1095,13 +1099,22 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, return -EINVAL; if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE) return -EINVAL; - if (copy_from_user(&ctl, &user_vmcb->control, sizeof(ctl))) - return -EFAULT; - if (copy_from_user(&save, &user_vmcb->save, sizeof(save))) - return -EFAULT; - if (!nested_vmcb_check_controls(&ctl)) - return -EINVAL; + ret = -ENOMEM; + ctl = kzalloc(sizeof(*ctl), GFP_KERNEL); + save = kzalloc(sizeof(*save), GFP_KERNEL); + if (!ctl || !save) + goto out_free; + + ret = -EFAULT; + if (copy_from_user(ctl, &user_vmcb->control, sizeof(*ctl))) + goto out_free; + if (copy_from_user(save, &user_vmcb->save, sizeof(*save))) + goto out_free; + + ret = -EINVAL; + if (!nested_vmcb_check_controls(ctl)) + goto out_free; /* * Processor state contains L2 state. Check that it is @@ -1109,15 +1122,15 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, */ cr0 = kvm_read_cr0(vcpu); if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW)) - return -EINVAL; + goto out_free; /* * Validate host state saved from before VMRUN (see * nested_svm_check_permissions). * TODO: validate reserved bits for all saved state. */ - if (!(save.cr0 & X86_CR0_PG)) - return -EINVAL; + if (!(save->cr0 & X86_CR0_PG)) + goto out_free; /* * All checks done, we can enter guest mode. L1 control fields @@ -1126,15 +1139,21 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, * contains saved L1 state. */ copy_vmcb_control_area(&hsave->control, &svm->vmcb->control); - hsave->save = save; + hsave->save = *save; svm->nested.vmcb = kvm_state->hdr.svm.vmcb_pa; - load_nested_vmcb_control(svm, &ctl); + load_nested_vmcb_control(svm, ctl); nested_prepare_vmcb_control(svm); out_set_gif: svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); - return 0; + + ret = 0; +out_free: + kfree(save); + kfree(ctl); + + return ret; } struct kvm_x86_nested_ops svm_nested_ops = { From d07f46f9f51afc7fc9f021eae19eba3c2e7870ac Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:03 +0200 Subject: [PATCH 015/265] KVM: SVM: Add GHCB definitions Extend the vmcb_safe_area with SEV-ES fields and add a new 'struct ghcb' which will be used for guest-hypervisor communication. Signed-off-by: Tom Lendacky Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-3-joro@8bytes.org --- arch/x86/include/asm/svm.h | 51 ++++++++++++++++++++++++++++++++++++-- arch/x86/kvm/svm/svm.c | 2 ++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index 8a1f5382a4ea..acac55d6f941 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -200,13 +200,60 @@ struct __attribute__ ((__packed__)) vmcb_save_area { u64 br_to; u64 last_excp_from; u64 last_excp_to; + + /* + * The following part of the save area is valid only for + * SEV-ES guests when referenced through the GHCB. + */ + u8 reserved_7[104]; + u64 reserved_8; /* rax already available at 0x01f8 */ + u64 rcx; + u64 rdx; + u64 rbx; + u64 reserved_9; /* rsp already available at 0x01d8 */ + u64 rbp; + u64 rsi; + u64 rdi; + u64 r8; + u64 r9; + u64 r10; + u64 r11; + u64 r12; + u64 r13; + u64 r14; + u64 r15; + u8 reserved_10[16]; + u64 sw_exit_code; + u64 sw_exit_info_1; + u64 sw_exit_info_2; + u64 sw_scratch; + u8 reserved_11[56]; + u64 xcr0; + u8 valid_bitmap[16]; + u64 x87_state_gpa; }; +struct ghcb { + struct vmcb_save_area save; + u8 reserved_save[2048 - sizeof(struct vmcb_save_area)]; + + u8 shared_buffer[2032]; + + u8 reserved_1[10]; + u16 protocol_version; /* negotiated SEV-ES/GHCB protocol version */ + u32 ghcb_usage; +} __packed; + + +#define EXPECTED_VMCB_SAVE_AREA_SIZE 1032 +#define EXPECTED_VMCB_CONTROL_AREA_SIZE 256 +#define EXPECTED_GHCB_SIZE PAGE_SIZE static inline void __unused_size_checks(void) { - BUILD_BUG_ON(sizeof(struct vmcb_save_area) != 0x298); - BUILD_BUG_ON(sizeof(struct vmcb_control_area) != 256); + BUILD_BUG_ON(sizeof(struct vmcb_save_area) != EXPECTED_VMCB_SAVE_AREA_SIZE); + BUILD_BUG_ON(sizeof(struct vmcb_control_area) != EXPECTED_VMCB_CONTROL_AREA_SIZE); + BUILD_BUG_ON(sizeof(struct ghcb) != EXPECTED_GHCB_SIZE); } struct __attribute__ ((__packed__)) vmcb { diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 0194336b64a4..1db4fdcb4906 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4164,6 +4164,8 @@ static struct kvm_x86_init_ops svm_init_ops __initdata = { static int __init svm_init(void) { + __unused_size_checks(); + return kvm_init(&svm_init_ops, sizeof(struct vcpu_svm), __alignof__(struct vcpu_svm), THIS_MODULE); } From 3702c2f4eed2188440f65ecdfc89165106fe565d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:04 +0200 Subject: [PATCH 016/265] KVM: SVM: Add GHCB Accessor functions Building a correct GHCB for the hypervisor requires setting valid bits in the GHCB. Simplify that process by providing accessor functions to set values and to update the valid bitmap and to check the valid bitmap in KVM. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-4-joro@8bytes.org --- arch/x86/include/asm/svm.h | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index acac55d6f941..06e52585aed3 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -345,4 +345,47 @@ struct __attribute__ ((__packed__)) vmcb { #define SVM_CR0_SELECTIVE_MASK (X86_CR0_TS | X86_CR0_MP) +/* GHCB Accessor functions */ + +#define GHCB_BITMAP_IDX(field) \ + (offsetof(struct vmcb_save_area, field) / sizeof(u64)) + +#define DEFINE_GHCB_ACCESSORS(field) \ + static inline bool ghcb_##field##_is_valid(const struct ghcb *ghcb) \ + { \ + return test_bit(GHCB_BITMAP_IDX(field), \ + (unsigned long *)&ghcb->save.valid_bitmap); \ + } \ + \ + static inline void ghcb_set_##field(struct ghcb *ghcb, u64 value) \ + { \ + __set_bit(GHCB_BITMAP_IDX(field), \ + (unsigned long *)&ghcb->save.valid_bitmap); \ + ghcb->save.field = value; \ + } + +DEFINE_GHCB_ACCESSORS(cpl) +DEFINE_GHCB_ACCESSORS(rip) +DEFINE_GHCB_ACCESSORS(rsp) +DEFINE_GHCB_ACCESSORS(rax) +DEFINE_GHCB_ACCESSORS(rcx) +DEFINE_GHCB_ACCESSORS(rdx) +DEFINE_GHCB_ACCESSORS(rbx) +DEFINE_GHCB_ACCESSORS(rbp) +DEFINE_GHCB_ACCESSORS(rsi) +DEFINE_GHCB_ACCESSORS(rdi) +DEFINE_GHCB_ACCESSORS(r8) +DEFINE_GHCB_ACCESSORS(r9) +DEFINE_GHCB_ACCESSORS(r10) +DEFINE_GHCB_ACCESSORS(r11) +DEFINE_GHCB_ACCESSORS(r12) +DEFINE_GHCB_ACCESSORS(r13) +DEFINE_GHCB_ACCESSORS(r14) +DEFINE_GHCB_ACCESSORS(r15) +DEFINE_GHCB_ACCESSORS(sw_exit_code) +DEFINE_GHCB_ACCESSORS(sw_exit_info_1) +DEFINE_GHCB_ACCESSORS(sw_exit_info_2) +DEFINE_GHCB_ACCESSORS(sw_scratch) +DEFINE_GHCB_ACCESSORS(xcr0) + #endif From 976bc5e2aceedef13e0ba1f0e6e372a22164aa0c Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Mon, 7 Sep 2020 15:15:05 +0200 Subject: [PATCH 017/265] KVM: SVM: Use __packed shorthand Use the shorthand to make it more readable. No functional changes. Signed-off-by: Borislav Petkov Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-5-joro@8bytes.org --- arch/x86/include/asm/svm.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index 06e52585aed3..cf13f9e78585 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -150,14 +150,14 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define SVM_NESTED_CTL_NP_ENABLE BIT(0) #define SVM_NESTED_CTL_SEV_ENABLE BIT(1) -struct __attribute__ ((__packed__)) vmcb_seg { +struct vmcb_seg { u16 selector; u16 attrib; u32 limit; u64 base; -}; +} __packed; -struct __attribute__ ((__packed__)) vmcb_save_area { +struct vmcb_save_area { struct vmcb_seg es; struct vmcb_seg cs; struct vmcb_seg ss; @@ -231,7 +231,7 @@ struct __attribute__ ((__packed__)) vmcb_save_area { u64 xcr0; u8 valid_bitmap[16]; u64 x87_state_gpa; -}; +} __packed; struct ghcb { struct vmcb_save_area save; @@ -256,11 +256,11 @@ static inline void __unused_size_checks(void) BUILD_BUG_ON(sizeof(struct ghcb) != EXPECTED_GHCB_SIZE); } -struct __attribute__ ((__packed__)) vmcb { +struct vmcb { struct vmcb_control_area control; u8 reserved_control[1024 - sizeof(struct vmcb_control_area)]; struct vmcb_save_area save; -}; +} __packed; #define SVM_CPUID_FUNC 0x8000000a From 360e7c5c4ca4fd8e627781ed42f95d58bc3bb732 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:06 +0200 Subject: [PATCH 018/265] x86/cpufeatures: Add SEV-ES CPU feature Add CPU feature detection for Secure Encrypted Virtualization with Encrypted State. This feature enhances SEV by also encrypting the guest register state, making it in-accessible to the hypervisor. Signed-off-by: Tom Lendacky Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-6-joro@8bytes.org --- arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/kernel/cpu/amd.c | 3 ++- arch/x86/kernel/cpu/scattered.c | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 83fc9d38eb1f..1205c1b6991a 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -236,6 +236,7 @@ #define X86_FEATURE_EPT_AD ( 8*32+17) /* Intel Extended Page Table access-dirty bit */ #define X86_FEATURE_VMCALL ( 8*32+18) /* "" Hypervisor supports the VMCALL instruction */ #define X86_FEATURE_VMW_VMMCALL ( 8*32+19) /* "" VMware prefers VMMCALL hypercall instruction */ +#define X86_FEATURE_SEV_ES ( 8*32+20) /* AMD Secure Encrypted Virtualization - Encrypted State */ /* Intel-defined CPU features, CPUID level 0x00000007:0 (EBX), word 9 */ #define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* RDFSBASE, WRFSBASE, RDGSBASE, WRGSBASE instructions*/ diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index dcc3d943c68f..6062ce586b95 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -614,7 +614,7 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c) * If BIOS has not enabled SME then don't advertise the * SME feature (set in scattered.c). * For SEV: If BIOS has not enabled SEV then don't advertise the - * SEV feature (set in scattered.c). + * SEV and SEV_ES feature (set in scattered.c). * * In all cases, since support for SME and SEV requires long mode, * don't advertise the feature under CONFIG_X86_32. @@ -645,6 +645,7 @@ clear_all: setup_clear_cpu_cap(X86_FEATURE_SME); clear_sev: setup_clear_cpu_cap(X86_FEATURE_SEV); + setup_clear_cpu_cap(X86_FEATURE_SEV_ES); } } diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 62b137c3c97a..30f354989cf1 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -41,6 +41,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, { X86_FEATURE_SME, CPUID_EAX, 0, 0x8000001f, 0 }, { X86_FEATURE_SEV, CPUID_EAX, 1, 0x8000001f, 0 }, + { X86_FEATURE_SEV_ES, CPUID_EAX, 3, 0x8000001f, 0 }, { 0, 0, 0, 0, 0 } }; From 05a2fdf3230306daee1def019b8f52cd06bd2e48 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:07 +0200 Subject: [PATCH 019/265] x86/traps: Move pf error codes to Move the definition of the x86 page-fault error code bits to a new header file asm/trap_pf.h. This makes it easier to include them into pre-decompression boot code. No functional changes. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-7-joro@8bytes.org --- arch/x86/include/asm/trap_pf.h | 24 ++++++++++++++++++++++++ arch/x86/include/asm/traps.h | 19 +------------------ 2 files changed, 25 insertions(+), 18 deletions(-) create mode 100644 arch/x86/include/asm/trap_pf.h diff --git a/arch/x86/include/asm/trap_pf.h b/arch/x86/include/asm/trap_pf.h new file mode 100644 index 000000000000..305bc1214aef --- /dev/null +++ b/arch/x86/include/asm/trap_pf.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_TRAP_PF_H +#define _ASM_X86_TRAP_PF_H + +/* + * Page fault error code bits: + * + * bit 0 == 0: no page found 1: protection fault + * bit 1 == 0: read access 1: write access + * bit 2 == 0: kernel-mode access 1: user-mode access + * bit 3 == 1: use of reserved bit detected + * bit 4 == 1: fault was an instruction fetch + * bit 5 == 1: protection keys block access + */ +enum x86_pf_error_code { + X86_PF_PROT = 1 << 0, + X86_PF_WRITE = 1 << 1, + X86_PF_USER = 1 << 2, + X86_PF_RSVD = 1 << 3, + X86_PF_INSTR = 1 << 4, + X86_PF_PK = 1 << 5, +}; + +#endif /* _ASM_X86_TRAP_PF_H */ diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index 714b1a30e7b0..6a308355ea29 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -8,6 +8,7 @@ #include #include #include /* TRAP_TRACE, ... */ +#include #ifdef CONFIG_X86_64 asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs); @@ -41,22 +42,4 @@ void __noreturn handle_stack_overflow(const char *message, unsigned long fault_address); #endif -/* - * Page fault error code bits: - * - * bit 0 == 0: no page found 1: protection fault - * bit 1 == 0: read access 1: write access - * bit 2 == 0: kernel-mode access 1: user-mode access - * bit 3 == 1: use of reserved bit detected - * bit 4 == 1: fault was an instruction fetch - * bit 5 == 1: protection keys block access - */ -enum x86_pf_error_code { - X86_PF_PROT = 1 << 0, - X86_PF_WRITE = 1 << 1, - X86_PF_USER = 1 << 2, - X86_PF_RSVD = 1 << 3, - X86_PF_INSTR = 1 << 4, - X86_PF_PK = 1 << 5, -}; #endif /* _ASM_X86_TRAPS_H */ From 05a2ae7c033ee30f25fbed3ceed549a5cac398a9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:08 +0200 Subject: [PATCH 020/265] x86/insn: Make inat-tables.c suitable for pre-decompression code The inat-tables.c file has some arrays in it that contain pointers to other arrays. These pointers need to be relocated when the kernel image is moved to a different location. The pre-decompression boot-code has no support for applying ELF relocations, so initialize these arrays at runtime in the pre-decompression code to make sure all pointers are correctly initialized. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Acked-by: Masami Hiramatsu Link: https://lkml.kernel.org/r/20200907131613.12703-8-joro@8bytes.org --- arch/x86/tools/gen-insn-attr-x86.awk | 50 +++++++++++++++++++++- tools/arch/x86/tools/gen-insn-attr-x86.awk | 50 +++++++++++++++++++++- 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/arch/x86/tools/gen-insn-attr-x86.awk b/arch/x86/tools/gen-insn-attr-x86.awk index a42015b305f4..af38469afd14 100644 --- a/arch/x86/tools/gen-insn-attr-x86.awk +++ b/arch/x86/tools/gen-insn-attr-x86.awk @@ -362,6 +362,9 @@ function convert_operands(count,opnd, i,j,imm,mod) END { if (awkchecked != "") exit 1 + + print "#ifndef __BOOT_COMPRESSED\n" + # print escape opcode map's array print "/* Escape opcode map array */" print "const insn_attr_t * const inat_escape_tables[INAT_ESC_MAX + 1]" \ @@ -388,6 +391,51 @@ END { for (j = 0; j < max_lprefix; j++) if (atable[i,j]) print " ["i"]["j"] = "atable[i,j]"," - print "};" + print "};\n" + + print "#else /* !__BOOT_COMPRESSED */\n" + + print "/* Escape opcode map array */" + print "static const insn_attr_t *inat_escape_tables[INAT_ESC_MAX + 1]" \ + "[INAT_LSTPFX_MAX + 1];" + print "" + + print "/* Group opcode map array */" + print "static const insn_attr_t *inat_group_tables[INAT_GRP_MAX + 1]"\ + "[INAT_LSTPFX_MAX + 1];" + print "" + + print "/* AVX opcode map array */" + print "static const insn_attr_t *inat_avx_tables[X86_VEX_M_MAX + 1]"\ + "[INAT_LSTPFX_MAX + 1];" + print "" + + print "static void inat_init_tables(void)" + print "{" + + # print escape opcode map's array + print "\t/* Print Escape opcode map array */" + for (i = 0; i < geid; i++) + for (j = 0; j < max_lprefix; j++) + if (etable[i,j]) + print "\tinat_escape_tables["i"]["j"] = "etable[i,j]";" + print "" + + # print group opcode map's array + print "\t/* Print Group opcode map array */" + for (i = 0; i < ggid; i++) + for (j = 0; j < max_lprefix; j++) + if (gtable[i,j]) + print "\tinat_group_tables["i"]["j"] = "gtable[i,j]";" + print "" + # print AVX opcode map's array + print "\t/* Print AVX opcode map array */" + for (i = 0; i < gaid; i++) + for (j = 0; j < max_lprefix; j++) + if (atable[i,j]) + print "\tinat_avx_tables["i"]["j"] = "atable[i,j]";" + + print "}" + print "#endif" } diff --git a/tools/arch/x86/tools/gen-insn-attr-x86.awk b/tools/arch/x86/tools/gen-insn-attr-x86.awk index a42015b305f4..af38469afd14 100644 --- a/tools/arch/x86/tools/gen-insn-attr-x86.awk +++ b/tools/arch/x86/tools/gen-insn-attr-x86.awk @@ -362,6 +362,9 @@ function convert_operands(count,opnd, i,j,imm,mod) END { if (awkchecked != "") exit 1 + + print "#ifndef __BOOT_COMPRESSED\n" + # print escape opcode map's array print "/* Escape opcode map array */" print "const insn_attr_t * const inat_escape_tables[INAT_ESC_MAX + 1]" \ @@ -388,6 +391,51 @@ END { for (j = 0; j < max_lprefix; j++) if (atable[i,j]) print " ["i"]["j"] = "atable[i,j]"," - print "};" + print "};\n" + + print "#else /* !__BOOT_COMPRESSED */\n" + + print "/* Escape opcode map array */" + print "static const insn_attr_t *inat_escape_tables[INAT_ESC_MAX + 1]" \ + "[INAT_LSTPFX_MAX + 1];" + print "" + + print "/* Group opcode map array */" + print "static const insn_attr_t *inat_group_tables[INAT_GRP_MAX + 1]"\ + "[INAT_LSTPFX_MAX + 1];" + print "" + + print "/* AVX opcode map array */" + print "static const insn_attr_t *inat_avx_tables[X86_VEX_M_MAX + 1]"\ + "[INAT_LSTPFX_MAX + 1];" + print "" + + print "static void inat_init_tables(void)" + print "{" + + # print escape opcode map's array + print "\t/* Print Escape opcode map array */" + for (i = 0; i < geid; i++) + for (j = 0; j < max_lprefix; j++) + if (etable[i,j]) + print "\tinat_escape_tables["i"]["j"] = "etable[i,j]";" + print "" + + # print group opcode map's array + print "\t/* Print Group opcode map array */" + for (i = 0; i < ggid; i++) + for (j = 0; j < max_lprefix; j++) + if (gtable[i,j]) + print "\tinat_group_tables["i"]["j"] = "gtable[i,j]";" + print "" + # print AVX opcode map's array + print "\t/* Print AVX opcode map array */" + for (i = 0; i < gaid; i++) + for (j = 0; j < max_lprefix; j++) + if (atable[i,j]) + print "\tinat_avx_tables["i"]["j"] = "atable[i,j]";" + + print "}" + print "#endif" } From 172b75e56b08846e6fb07a88e5685ce4e24f4620 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:09 +0200 Subject: [PATCH 021/265] x86/umip: Factor out instruction fetch Factor out the code to fetch the instruction from user-space to a helper function. No functional changes. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-9-joro@8bytes.org --- arch/x86/include/asm/insn-eval.h | 2 ++ arch/x86/kernel/umip.c | 26 +++++----------------- arch/x86/lib/insn-eval.c | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h index 2b6ccf2c49f1..b8b9ef1bbd06 100644 --- a/arch/x86/include/asm/insn-eval.h +++ b/arch/x86/include/asm/insn-eval.h @@ -19,5 +19,7 @@ void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs); int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs); unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx); int insn_get_code_seg_params(struct pt_regs *regs); +int insn_fetch_from_user(struct pt_regs *regs, + unsigned char buf[MAX_INSN_SIZE]); #endif /* _ASM_X86_INSN_EVAL_H */ diff --git a/arch/x86/kernel/umip.c b/arch/x86/kernel/umip.c index 2c304fd0bb1a..ad135be4f1f0 100644 --- a/arch/x86/kernel/umip.c +++ b/arch/x86/kernel/umip.c @@ -335,11 +335,11 @@ static void force_sig_info_umip_fault(void __user *addr, struct pt_regs *regs) */ bool fixup_umip_exception(struct pt_regs *regs) { - int not_copied, nr_copied, reg_offset, dummy_data_size, umip_inst; - unsigned long seg_base = 0, *reg_addr; + int nr_copied, reg_offset, dummy_data_size, umip_inst; /* 10 bytes is the maximum size of the result of UMIP instructions */ unsigned char dummy_data[10] = { 0 }; unsigned char buf[MAX_INSN_SIZE]; + unsigned long *reg_addr; void __user *uaddr; struct insn insn; int seg_defs; @@ -347,26 +347,12 @@ bool fixup_umip_exception(struct pt_regs *regs) if (!regs) return false; - /* - * If not in user-space long mode, a custom code segment could be in - * use. This is true in protected mode (if the process defined a local - * descriptor table), or virtual-8086 mode. In most of the cases - * seg_base will be zero as in USER_CS. - */ - if (!user_64bit_mode(regs)) - seg_base = insn_get_seg_base(regs, INAT_SEG_REG_CS); - - if (seg_base == -1L) - return false; - - not_copied = copy_from_user(buf, (void __user *)(seg_base + regs->ip), - sizeof(buf)); - nr_copied = sizeof(buf) - not_copied; + nr_copied = insn_fetch_from_user(regs, buf); /* - * The copy_from_user above could have failed if user code is protected - * by a memory protection key. Give up on emulation in such a case. - * Should we issue a page fault? + * The insn_fetch_from_user above could have failed if user code + * is protected by a memory protection key. Give up on emulation + * in such a case. Should we issue a page fault? */ if (!nr_copied) return false; diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index 5e69603ff63f..947b7f1a0042 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -1367,3 +1367,41 @@ void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs) return (void __user *)-1L; } } + +/** + * insn_fetch_from_user() - Copy instruction bytes from user-space memory + * @regs: Structure with register values as seen when entering kernel mode + * @buf: Array to store the fetched instruction + * + * Gets the linear address of the instruction and copies the instruction bytes + * to the buf. + * + * Returns: + * + * Number of instruction bytes copied. + * + * 0 if nothing was copied. + */ +int insn_fetch_from_user(struct pt_regs *regs, unsigned char buf[MAX_INSN_SIZE]) +{ + unsigned long seg_base = 0; + int not_copied; + + /* + * If not in user-space long mode, a custom code segment could be in + * use. This is true in protected mode (if the process defined a local + * descriptor table), or virtual-8086 mode. In most of the cases + * seg_base will be zero as in USER_CS. + */ + if (!user_64bit_mode(regs)) { + seg_base = insn_get_seg_base(regs, INAT_SEG_REG_CS); + if (seg_base == -1L) + return 0; + } + + + not_copied = copy_from_user(buf, (void __user *)(seg_base + regs->ip), + MAX_INSN_SIZE); + + return MAX_INSN_SIZE - not_copied; +} From 172639d79977ca7b5ce6f84f6606262f4081718f Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:10 +0200 Subject: [PATCH 022/265] x86/umip: Factor out instruction decoding Factor out the code used to decode an instruction with the correct address and operand sizes to a helper function. No functional changes. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-10-joro@8bytes.org --- arch/x86/include/asm/insn-eval.h | 2 ++ arch/x86/kernel/umip.c | 23 +--------------- arch/x86/lib/insn-eval.c | 45 ++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h index b8b9ef1bbd06..392b4fe377f9 100644 --- a/arch/x86/include/asm/insn-eval.h +++ b/arch/x86/include/asm/insn-eval.h @@ -21,5 +21,7 @@ unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx); int insn_get_code_seg_params(struct pt_regs *regs); int insn_fetch_from_user(struct pt_regs *regs, unsigned char buf[MAX_INSN_SIZE]); +bool insn_decode(struct insn *insn, struct pt_regs *regs, + unsigned char buf[MAX_INSN_SIZE], int buf_size); #endif /* _ASM_X86_INSN_EVAL_H */ diff --git a/arch/x86/kernel/umip.c b/arch/x86/kernel/umip.c index ad135be4f1f0..f6225bf22c02 100644 --- a/arch/x86/kernel/umip.c +++ b/arch/x86/kernel/umip.c @@ -342,7 +342,6 @@ bool fixup_umip_exception(struct pt_regs *regs) unsigned long *reg_addr; void __user *uaddr; struct insn insn; - int seg_defs; if (!regs) return false; @@ -357,27 +356,7 @@ bool fixup_umip_exception(struct pt_regs *regs) if (!nr_copied) return false; - insn_init(&insn, buf, nr_copied, user_64bit_mode(regs)); - - /* - * Override the default operand and address sizes with what is specified - * in the code segment descriptor. The instruction decoder only sets - * the address size it to either 4 or 8 address bytes and does nothing - * for the operand bytes. This OK for most of the cases, but we could - * have special cases where, for instance, a 16-bit code segment - * descriptor is used. - * If there is an address override prefix, the instruction decoder - * correctly updates these values, even for 16-bit defaults. - */ - seg_defs = insn_get_code_seg_params(regs); - if (seg_defs == -EINVAL) - return false; - - insn.addr_bytes = INSN_CODE_SEG_ADDR_SZ(seg_defs); - insn.opnd_bytes = INSN_CODE_SEG_OPND_SZ(seg_defs); - - insn_get_length(&insn); - if (nr_copied < insn.length) + if (!insn_decode(&insn, regs, buf, nr_copied)) return false; umip_inst = identify_insn(&insn); diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index 947b7f1a0042..2323c85132cf 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -1405,3 +1405,48 @@ int insn_fetch_from_user(struct pt_regs *regs, unsigned char buf[MAX_INSN_SIZE]) return MAX_INSN_SIZE - not_copied; } + +/** + * insn_decode() - Decode an instruction + * @insn: Structure to store decoded instruction + * @regs: Structure with register values as seen when entering kernel mode + * @buf: Buffer containing the instruction bytes + * @buf_size: Number of instruction bytes available in buf + * + * Decodes the instruction provided in buf and stores the decoding results in + * insn. Also determines the correct address and operand sizes. + * + * Returns: + * + * True if instruction was decoded, False otherwise. + */ +bool insn_decode(struct insn *insn, struct pt_regs *regs, + unsigned char buf[MAX_INSN_SIZE], int buf_size) +{ + int seg_defs; + + insn_init(insn, buf, buf_size, user_64bit_mode(regs)); + + /* + * Override the default operand and address sizes with what is specified + * in the code segment descriptor. The instruction decoder only sets + * the address size it to either 4 or 8 address bytes and does nothing + * for the operand bytes. This OK for most of the cases, but we could + * have special cases where, for instance, a 16-bit code segment + * descriptor is used. + * If there is an address override prefix, the instruction decoder + * correctly updates these values, even for 16-bit defaults. + */ + seg_defs = insn_get_code_seg_params(regs); + if (seg_defs == -EINVAL) + return false; + + insn->addr_bytes = INSN_CODE_SEG_ADDR_SZ(seg_defs); + insn->opnd_bytes = INSN_CODE_SEG_OPND_SZ(seg_defs); + + insn_get_length(insn); + if (buf_size < insn->length) + return false; + + return true; +} From 7af1bd822dd45a669fc178a35cc8183922333d56 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:11 +0200 Subject: [PATCH 023/265] x86/insn: Add insn_get_modrm_reg_off() Add a function to the instruction decoder which returns the pt_regs offset of the register specified in the reg field of the modrm byte. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Acked-by: Masami Hiramatsu Link: https://lkml.kernel.org/r/20200907131613.12703-11-joro@8bytes.org --- arch/x86/include/asm/insn-eval.h | 1 + arch/x86/lib/insn-eval.c | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h index 392b4fe377f9..f748f57f1491 100644 --- a/arch/x86/include/asm/insn-eval.h +++ b/arch/x86/include/asm/insn-eval.h @@ -17,6 +17,7 @@ void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs); int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs); +int insn_get_modrm_reg_off(struct insn *insn, struct pt_regs *regs); unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx); int insn_get_code_seg_params(struct pt_regs *regs); int insn_fetch_from_user(struct pt_regs *regs, diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index 2323c85132cf..f20942ce76c6 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -20,6 +20,7 @@ enum reg_type { REG_TYPE_RM = 0, + REG_TYPE_REG, REG_TYPE_INDEX, REG_TYPE_BASE, }; @@ -439,6 +440,13 @@ static int get_reg_offset(struct insn *insn, struct pt_regs *regs, regno += 8; break; + case REG_TYPE_REG: + regno = X86_MODRM_REG(insn->modrm.value); + + if (X86_REX_R(insn->rex_prefix.value)) + regno += 8; + break; + case REG_TYPE_INDEX: regno = X86_SIB_INDEX(insn->sib.value); if (X86_REX_X(insn->rex_prefix.value)) @@ -807,6 +815,21 @@ int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs) return get_reg_offset(insn, regs, REG_TYPE_RM); } +/** + * insn_get_modrm_reg_off() - Obtain register in reg part of the ModRM byte + * @insn: Instruction containing the ModRM byte + * @regs: Register values as seen when entering kernel mode + * + * Returns: + * + * The register indicated by the reg part of the ModRM byte. The + * register is obtained as an offset from the base of pt_regs. + */ +int insn_get_modrm_reg_off(struct insn *insn, struct pt_regs *regs) +{ + return get_reg_offset(insn, regs, REG_TYPE_REG); +} + /** * get_seg_base_limit() - obtain base address and limit of a segment * @insn: Instruction. Must be valid. From 5901781a11175a5e5ee91746ec8627f18d47eebd Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:12 +0200 Subject: [PATCH 024/265] x86/insn: Add insn_has_rep_prefix() helper Add a function to check whether an instruction has a REP prefix. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Masami Hiramatsu Link: https://lkml.kernel.org/r/20200907131613.12703-12-joro@8bytes.org --- arch/x86/include/asm/insn-eval.h | 1 + arch/x86/lib/insn-eval.c | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h index f748f57f1491..a0f839aa144d 100644 --- a/arch/x86/include/asm/insn-eval.h +++ b/arch/x86/include/asm/insn-eval.h @@ -15,6 +15,7 @@ #define INSN_CODE_SEG_OPND_SZ(params) (params & 0xf) #define INSN_CODE_SEG_PARAMS(oper_sz, addr_sz) (oper_sz | (addr_sz << 4)) +bool insn_has_rep_prefix(struct insn *insn); void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs); int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs); int insn_get_modrm_reg_off(struct insn *insn, struct pt_regs *regs); diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index f20942ce76c6..58f7fb95c7f4 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -53,6 +53,30 @@ static bool is_string_insn(struct insn *insn) } } +/** + * insn_has_rep_prefix() - Determine if instruction has a REP prefix + * @insn: Instruction containing the prefix to inspect + * + * Returns: + * + * true if the instruction has a REP prefix, false if not. + */ +bool insn_has_rep_prefix(struct insn *insn) +{ + int i; + + insn_get_prefixes(insn); + + for (i = 0; i < insn->prefixes.nbytes; i++) { + insn_byte_t p = insn->prefixes.bytes[i]; + + if (p == 0xf2 || p == 0xf3) + return true; + } + + return false; +} + /** * get_seg_reg_override_idx() - obtain segment register override index * @insn: Valid instruction with segment override prefixes From 6ba0efa46047936afa81460489cfd24bc95dd863 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:13 +0200 Subject: [PATCH 025/265] x86/boot/compressed/64: Disable red-zone usage The x86-64 ABI defines a red-zone on the stack: The 128-byte area beyond the location pointed to by %rsp is considered to be reserved and shall not be modified by signal or interrupt handlers. Therefore, functions may use this area for temporary data that is not needed across function calls. In particular, leaf functions may use this area for their entire stack frame, rather than adjusting the stack pointer in the prologue and epilogue. This area is known as the red zone. This is not compatible with exception handling, because the IRET frame written by the hardware at the stack pointer and the functions to handle the exception will overwrite the temporary variables of the interrupted function, causing undefined behavior. So disable red-zones for the pre-decompression boot code. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-13-joro@8bytes.org --- arch/x86/boot/compressed/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 3962f592633d..5343079af973 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -32,7 +32,7 @@ KBUILD_CFLAGS := -m$(BITS) -O2 KBUILD_CFLAGS += -fno-strict-aliasing $(call cc-option, -fPIE, -fPIC) KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING cflags-$(CONFIG_X86_32) := -march=i386 -cflags-$(CONFIG_X86_64) := -mcmodel=small +cflags-$(CONFIG_X86_64) := -mcmodel=small -mno-red-zone KBUILD_CFLAGS += $(cflags-y) KBUILD_CFLAGS += -mno-mmx -mno-sse KBUILD_CFLAGS += -ffreestanding From 64e682638eb51070ba6044535b250aad43c5564e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:14 +0200 Subject: [PATCH 026/265] x86/boot/compressed/64: Add IDT Infrastructure Add code needed to setup an IDT in the early pre-decompression boot-code. The IDT is loaded first in startup_64, which is after EfiExitBootServices() has been called, and later reloaded when the kernel image has been relocated to the end of the decompression area. This allows to setup different IDT handlers before and after the relocation. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-14-joro@8bytes.org --- arch/x86/boot/compressed/Makefile | 1 + arch/x86/boot/compressed/head_64.S | 25 +++++++- arch/x86/boot/compressed/idt_64.c | 44 ++++++++++++++ arch/x86/boot/compressed/idt_handlers_64.S | 70 ++++++++++++++++++++++ arch/x86/boot/compressed/misc.h | 5 ++ arch/x86/include/asm/desc_defs.h | 3 + 6 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 arch/x86/boot/compressed/idt_64.c create mode 100644 arch/x86/boot/compressed/idt_handlers_64.S diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 5343079af973..c661dc57674e 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -85,6 +85,7 @@ vmlinux-objs-$(CONFIG_EARLY_PRINTK) += $(obj)/early_serial_console.o vmlinux-objs-$(CONFIG_RANDOMIZE_BASE) += $(obj)/kaslr.o ifdef CONFIG_X86_64 vmlinux-objs-$(CONFIG_RANDOMIZE_BASE) += $(obj)/kaslr_64.o + vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o vmlinux-objs-y += $(obj)/mem_encrypt.o vmlinux-objs-y += $(obj)/pgtable_64.o endif diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 97d37f0a34f5..c634ed8636da 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -33,6 +33,7 @@ #include #include #include +#include #include "pgtable.h" /* @@ -410,6 +411,10 @@ SYM_CODE_START(startup_64) .Lon_kernel_cs: + pushq %rsi + call load_stage1_idt + popq %rsi + /* * paging_prepare() sets up the trampoline and checks if we need to * enable 5-level paging. @@ -537,6 +542,13 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated) shrq $3, %rcx rep stosq +/* + * Load stage2 IDT + */ + pushq %rsi + call load_stage2_idt + popq %rsi + /* * Do the extraction, and jump to the new kernel.. */ @@ -690,10 +702,21 @@ SYM_DATA_START_LOCAL(gdt) .quad 0x0000000000000000 /* TS continued */ SYM_DATA_END_LABEL(gdt, SYM_L_LOCAL, gdt_end) +SYM_DATA_START(boot_idt_desc) + .word boot_idt_end - boot_idt - 1 + .quad 0 +SYM_DATA_END(boot_idt_desc) + .balign 8 +SYM_DATA_START(boot_idt) + .rept BOOT_IDT_ENTRIES + .quad 0 + .quad 0 + .endr +SYM_DATA_END_LABEL(boot_idt, SYM_L_GLOBAL, boot_idt_end) + #ifdef CONFIG_EFI_STUB SYM_DATA(image_offset, .long 0) #endif - #ifdef CONFIG_EFI_MIXED SYM_DATA_LOCAL(efi32_boot_args, .long 0, 0, 0) SYM_DATA(efi_is64, .byte 1) diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c new file mode 100644 index 000000000000..082cd6bca033 --- /dev/null +++ b/arch/x86/boot/compressed/idt_64.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include "misc.h" + +static void set_idt_entry(int vector, void (*handler)(void)) +{ + unsigned long address = (unsigned long)handler; + gate_desc entry; + + memset(&entry, 0, sizeof(entry)); + + entry.offset_low = (u16)(address & 0xffff); + entry.segment = __KERNEL_CS; + entry.bits.type = GATE_TRAP; + entry.bits.p = 1; + entry.offset_middle = (u16)((address >> 16) & 0xffff); + entry.offset_high = (u32)(address >> 32); + + memcpy(&boot_idt[vector], &entry, sizeof(entry)); +} + +/* Have this here so we don't need to include */ +static void load_boot_idt(const struct desc_ptr *dtr) +{ + asm volatile("lidt %0"::"m" (*dtr)); +} + +/* Setup IDT before kernel jumping to .Lrelocated */ +void load_stage1_idt(void) +{ + boot_idt_desc.address = (unsigned long)boot_idt; + + load_boot_idt(&boot_idt_desc); +} + +/* Setup IDT after kernel jumping to .Lrelocated */ +void load_stage2_idt(void) +{ + boot_idt_desc.address = (unsigned long)boot_idt; + + load_boot_idt(&boot_idt_desc); +} diff --git a/arch/x86/boot/compressed/idt_handlers_64.S b/arch/x86/boot/compressed/idt_handlers_64.S new file mode 100644 index 000000000000..36dee2f40a8b --- /dev/null +++ b/arch/x86/boot/compressed/idt_handlers_64.S @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Early IDT handler entry points + * + * Copyright (C) 2019 SUSE + * + * Author: Joerg Roedel + */ + +#include + +/* For ORIG_RAX */ +#include "../../entry/calling.h" + +.macro EXCEPTION_HANDLER name function error_code=0 +SYM_FUNC_START(\name) + + /* Build pt_regs */ + .if \error_code == 0 + pushq $0 + .endif + + pushq %rdi + pushq %rsi + pushq %rdx + pushq %rcx + pushq %rax + pushq %r8 + pushq %r9 + pushq %r10 + pushq %r11 + pushq %rbx + pushq %rbp + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + + /* Call handler with pt_regs */ + movq %rsp, %rdi + /* Error code is second parameter */ + movq ORIG_RAX(%rsp), %rsi + call \function + + /* Restore regs */ + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbp + popq %rbx + popq %r11 + popq %r10 + popq %r9 + popq %r8 + popq %rax + popq %rcx + popq %rdx + popq %rsi + popq %rdi + + /* Remove error code and return */ + addq $8, %rsp + + iretq +SYM_FUNC_END(\name) + .endm + + .text + .code64 diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 3efce27ba35c..8feb5f6f329e 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -23,6 +23,7 @@ #include #include #include +#include #define BOOT_CTYPE_H #include @@ -133,4 +134,8 @@ int count_immovable_mem_regions(void); static inline int count_immovable_mem_regions(void) { return 0; } #endif +/* idt_64.c */ +extern gate_desc boot_idt[BOOT_IDT_ENTRIES]; +extern struct desc_ptr boot_idt_desc; + #endif /* BOOT_COMPRESSED_MISC_H */ diff --git a/arch/x86/include/asm/desc_defs.h b/arch/x86/include/asm/desc_defs.h index a91f3b6e4f2a..5621fb3f2d1a 100644 --- a/arch/x86/include/asm/desc_defs.h +++ b/arch/x86/include/asm/desc_defs.h @@ -109,6 +109,9 @@ struct desc_ptr { #endif /* !__ASSEMBLY__ */ +/* Boot IDT definitions */ +#define BOOT_IDT_ENTRIES 32 + /* Access rights as returned by LAR */ #define AR_TYPE_RODATA (0 * (1 << 9)) #define AR_TYPE_RWDATA (1 * (1 << 9)) From 5f2bb01682b7b067783207994c7b8a3dbeb1cd83 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:15 +0200 Subject: [PATCH 027/265] x86/boot/compressed/64: Rename kaslr_64.c to ident_map_64.c The file contains only code related to identity-mapped page tables. Rename the file and compile it always in. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-15-joro@8bytes.org --- arch/x86/boot/compressed/Makefile | 2 +- arch/x86/boot/compressed/{kaslr_64.c => ident_map_64.c} | 9 +++++++++ arch/x86/boot/compressed/kaslr.c | 9 --------- arch/x86/boot/compressed/misc.h | 8 ++++++++ 4 files changed, 18 insertions(+), 10 deletions(-) rename arch/x86/boot/compressed/{kaslr_64.c => ident_map_64.c} (95%) diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index c661dc57674e..e7f3eba99ea2 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -84,7 +84,7 @@ vmlinux-objs-y := $(obj)/vmlinux.lds $(obj)/kernel_info.o $(obj)/head_$(BITS).o vmlinux-objs-$(CONFIG_EARLY_PRINTK) += $(obj)/early_serial_console.o vmlinux-objs-$(CONFIG_RANDOMIZE_BASE) += $(obj)/kaslr.o ifdef CONFIG_X86_64 - vmlinux-objs-$(CONFIG_RANDOMIZE_BASE) += $(obj)/kaslr_64.o + vmlinux-objs-y += $(obj)/ident_map_64.o vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o vmlinux-objs-y += $(obj)/mem_encrypt.o vmlinux-objs-y += $(obj)/pgtable_64.o diff --git a/arch/x86/boot/compressed/kaslr_64.c b/arch/x86/boot/compressed/ident_map_64.c similarity index 95% rename from arch/x86/boot/compressed/kaslr_64.c rename to arch/x86/boot/compressed/ident_map_64.c index f9c5c13d979b..d9932a133ac9 100644 --- a/arch/x86/boot/compressed/kaslr_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -29,6 +29,15 @@ #define __PAGE_OFFSET __PAGE_OFFSET_BASE #include "../../mm/ident_map.c" +#ifdef CONFIG_X86_5LEVEL +unsigned int __pgtable_l5_enabled; +unsigned int pgdir_shift = 39; +unsigned int ptrs_per_p4d = 1; +#endif + +/* Used by PAGE_KERN* macros: */ +pteval_t __default_kernel_pte_mask __read_mostly = ~0; + /* Used to track our page table allocation area. */ struct alloc_pgt_data { unsigned char *pgt_buf; diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index 877970d76249..e27de98ed038 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -40,17 +40,8 @@ #include /* For COMMAND_LINE_SIZE */ #undef _SETUP -#ifdef CONFIG_X86_5LEVEL -unsigned int __pgtable_l5_enabled; -unsigned int pgdir_shift __ro_after_init = 39; -unsigned int ptrs_per_p4d __ro_after_init = 1; -#endif - extern unsigned long get_cmd_line_ptr(void); -/* Used by PAGE_KERN* macros: */ -pteval_t __default_kernel_pte_mask __read_mostly = ~0; - /* Simplified build-specific string for starting entropy. */ static const char build_str[] = UTS_RELEASE " (" LINUX_COMPILE_BY "@" LINUX_COMPILE_HOST ") (" LINUX_COMPILER ") " UTS_VERSION; diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 8feb5f6f329e..98b7a1df9c59 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -134,6 +134,14 @@ int count_immovable_mem_regions(void); static inline int count_immovable_mem_regions(void) { return 0; } #endif +/* ident_map_64.c */ +#ifdef CONFIG_X86_5LEVEL +extern unsigned int __pgtable_l5_enabled, pgdir_shift, ptrs_per_p4d; +#endif + +/* Used by PAGE_KERN* macros: */ +extern pteval_t __default_kernel_pte_mask; + /* idt_64.c */ extern gate_desc boot_idt[BOOT_IDT_ENTRIES]; extern struct desc_ptr boot_idt_desc; From 8b0d3b3b41ab6f14f1ce6d4a6b1c5f60b825123f Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:16 +0200 Subject: [PATCH 028/265] x86/boot/compressed/64: Add page-fault handler Install a page-fault handler to add an identity mapping to addresses not yet mapped. Also do some checking whether the error code is sane. This makes non SEV-ES machines use the exception handling infrastructure in the pre-decompressions boot code too, making it less likely to break in the future. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-16-joro@8bytes.org --- arch/x86/boot/compressed/ident_map_64.c | 39 ++++++++++++++++++++++ arch/x86/boot/compressed/idt_64.c | 2 ++ arch/x86/boot/compressed/idt_handlers_64.S | 2 ++ arch/x86/boot/compressed/misc.h | 6 ++++ 4 files changed, 49 insertions(+) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index d9932a133ac9..e3d980ae9c2b 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -19,10 +19,13 @@ /* No PAGE_TABLE_ISOLATION support needed either: */ #undef CONFIG_PAGE_TABLE_ISOLATION +#include "error.h" #include "misc.h" /* These actually do the work of building the kernel identity maps. */ #include +#include +#include #include /* Use the static base for this part of the boot process */ #undef __PAGE_OFFSET @@ -160,3 +163,39 @@ void finalize_identity_maps(void) { write_cr3(top_level_pgt); } + +static void do_pf_error(const char *msg, unsigned long error_code, + unsigned long address, unsigned long ip) +{ + error_putstr(msg); + + error_putstr("\nError Code: "); + error_puthex(error_code); + error_putstr("\nCR2: 0x"); + error_puthex(address); + error_putstr("\nRIP relative to _head: 0x"); + error_puthex(ip - (unsigned long)_head); + error_putstr("\n"); + + error("Stopping.\n"); +} + +void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code) +{ + unsigned long address = native_read_cr2(); + + /* + * Check for unexpected error codes. Unexpected are: + * - Faults on present pages + * - User faults + * - Reserved bits set + */ + if (error_code & (X86_PF_PROT | X86_PF_USER | X86_PF_RSVD)) + do_pf_error("Unexpected page-fault:", error_code, address, regs->ip); + + /* + * Error code is sane - now identity map the 2M region around + * the faulting address. + */ + add_identity_map(address & PMD_MASK, PMD_SIZE); +} diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c index 082cd6bca033..5f083092a86d 100644 --- a/arch/x86/boot/compressed/idt_64.c +++ b/arch/x86/boot/compressed/idt_64.c @@ -40,5 +40,7 @@ void load_stage2_idt(void) { boot_idt_desc.address = (unsigned long)boot_idt; + set_idt_entry(X86_TRAP_PF, boot_page_fault); + load_boot_idt(&boot_idt_desc); } diff --git a/arch/x86/boot/compressed/idt_handlers_64.S b/arch/x86/boot/compressed/idt_handlers_64.S index 36dee2f40a8b..b20e57504a94 100644 --- a/arch/x86/boot/compressed/idt_handlers_64.S +++ b/arch/x86/boot/compressed/idt_handlers_64.S @@ -68,3 +68,5 @@ SYM_FUNC_END(\name) .text .code64 + +EXCEPTION_HANDLER boot_page_fault do_boot_page_fault error_code=1 diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 98b7a1df9c59..f0e199174c5f 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -37,6 +37,9 @@ #define memptr unsigned #endif +/* boot/compressed/vmlinux start and end markers */ +extern char _head[], _end[]; + /* misc.c */ extern memptr free_mem_ptr; extern memptr free_mem_end_ptr; @@ -146,4 +149,7 @@ extern pteval_t __default_kernel_pte_mask; extern gate_desc boot_idt[BOOT_IDT_ENTRIES]; extern struct desc_ptr boot_idt_desc; +/* IDT Entry Points */ +void boot_page_fault(void); + #endif /* BOOT_COMPRESSED_MISC_H */ From ca0e22d4f011a56e974fa3a712d76e86a791559d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:17 +0200 Subject: [PATCH 029/265] x86/boot/compressed/64: Always switch to own page table When booted through startup_64(), the kernel keeps running on the EFI page table until the KASLR code sets up its own page table. Without KASLR, the pre-decompression boot code never switches off the EFI page table. Change that by unconditionally switching to a kernel-controlled page table after relocation. This makes sure the kernel can make changes to the mapping when necessary, for example map pages unencrypted in SEV and SEV-ES guests. Also, remove the debug_putstr() calls in initialize_identity_maps() because the function now runs before console_init() is called. [ bp: Massage commit message. ] Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-17-joro@8bytes.org --- arch/x86/boot/compressed/head_64.S | 3 +- arch/x86/boot/compressed/ident_map_64.c | 51 +++++++++++++++---------- arch/x86/boot/compressed/kaslr.c | 3 -- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index c634ed8636da..fb6c0392306b 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -543,10 +543,11 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated) rep stosq /* - * Load stage2 IDT + * Load stage2 IDT and switch to our own page-table */ pushq %rsi call load_stage2_idt + call initialize_identity_maps popq %rsi /* diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index e3d980ae9c2b..ecf9353b064d 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -86,9 +86,31 @@ phys_addr_t physical_mask = (1ULL << __PHYSICAL_MASK_SHIFT) - 1; */ static struct x86_mapping_info mapping_info; +/* + * Adds the specified range to what will become the new identity mappings. + * Once all ranges have been added, the new mapping is activated by calling + * finalize_identity_maps() below. + */ +void add_identity_map(unsigned long start, unsigned long size) +{ + unsigned long end = start + size; + + /* Align boundary to 2M. */ + start = round_down(start, PMD_SIZE); + end = round_up(end, PMD_SIZE); + if (start >= end) + return; + + /* Build the mapping. */ + kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, + start, end); +} + /* Locates and clears a region for a new top level page table. */ void initialize_identity_maps(void) { + unsigned long start, size; + /* If running as an SEV guest, the encryption mask is required. */ set_sev_encryption_mask(); @@ -121,37 +143,24 @@ void initialize_identity_maps(void) */ top_level_pgt = read_cr3_pa(); if (p4d_offset((pgd_t *)top_level_pgt, 0) == (p4d_t *)_pgtable) { - debug_putstr("booted via startup_32()\n"); pgt_data.pgt_buf = _pgtable + BOOT_INIT_PGT_SIZE; pgt_data.pgt_buf_size = BOOT_PGT_SIZE - BOOT_INIT_PGT_SIZE; memset(pgt_data.pgt_buf, 0, pgt_data.pgt_buf_size); } else { - debug_putstr("booted via startup_64()\n"); pgt_data.pgt_buf = _pgtable; pgt_data.pgt_buf_size = BOOT_PGT_SIZE; memset(pgt_data.pgt_buf, 0, pgt_data.pgt_buf_size); top_level_pgt = (unsigned long)alloc_pgt_page(&pgt_data); } -} -/* - * Adds the specified range to what will become the new identity mappings. - * Once all ranges have been added, the new mapping is activated by calling - * finalize_identity_maps() below. - */ -void add_identity_map(unsigned long start, unsigned long size) -{ - unsigned long end = start + size; - - /* Align boundary to 2M. */ - start = round_down(start, PMD_SIZE); - end = round_up(end, PMD_SIZE); - if (start >= end) - return; - - /* Build the mapping. */ - kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, - start, end); + /* + * New page-table is set up - map the kernel image and load it + * into cr3. + */ + start = (unsigned long)_head; + size = _end - _head; + add_identity_map(start, size); + write_cr3(top_level_pgt); } /* diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index e27de98ed038..82662869c4cb 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -861,9 +861,6 @@ void choose_random_location(unsigned long input, boot_params->hdr.loadflags |= KASLR_FLAG; - /* Prepare to add new identity pagetables on demand. */ - initialize_identity_maps(); - if (IS_ENABLED(CONFIG_X86_32)) mem_limit = KERNEL_IMAGE_SIZE; else From 8570978ea030757839747aa9944ea576708be3d4 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:18 +0200 Subject: [PATCH 030/265] x86/boot/compressed/64: Don't pre-map memory in KASLR code With the page-fault handler in place, he identity mapping can be built on-demand. So remove the code which manually creates the mappings and unexport/remove the functions used for it. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-18-joro@8bytes.org --- arch/x86/boot/compressed/ident_map_64.c | 6 ++---- arch/x86/boot/compressed/kaslr.c | 24 +----------------------- arch/x86/boot/compressed/misc.h | 10 ---------- 3 files changed, 3 insertions(+), 37 deletions(-) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index ecf9353b064d..c63257bf8373 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -87,11 +87,9 @@ phys_addr_t physical_mask = (1ULL << __PHYSICAL_MASK_SHIFT) - 1; static struct x86_mapping_info mapping_info; /* - * Adds the specified range to what will become the new identity mappings. - * Once all ranges have been added, the new mapping is activated by calling - * finalize_identity_maps() below. + * Adds the specified range to the identity mappings. */ -void add_identity_map(unsigned long start, unsigned long size) +static void add_identity_map(unsigned long start, unsigned long size) { unsigned long end = start + size; diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index 82662869c4cb..b59547ce5b19 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -397,8 +397,6 @@ static void mem_avoid_init(unsigned long input, unsigned long input_size, */ mem_avoid[MEM_AVOID_ZO_RANGE].start = input; mem_avoid[MEM_AVOID_ZO_RANGE].size = (output + init_size) - input; - add_identity_map(mem_avoid[MEM_AVOID_ZO_RANGE].start, - mem_avoid[MEM_AVOID_ZO_RANGE].size); /* Avoid initrd. */ initrd_start = (u64)boot_params->ext_ramdisk_image << 32; @@ -416,15 +414,11 @@ static void mem_avoid_init(unsigned long input, unsigned long input_size, cmd_line_size = strnlen((char *)cmd_line, COMMAND_LINE_SIZE-1) + 1; mem_avoid[MEM_AVOID_CMDLINE].start = cmd_line; mem_avoid[MEM_AVOID_CMDLINE].size = cmd_line_size; - add_identity_map(mem_avoid[MEM_AVOID_CMDLINE].start, - mem_avoid[MEM_AVOID_CMDLINE].size); } /* Avoid boot parameters. */ mem_avoid[MEM_AVOID_BOOTPARAMS].start = (unsigned long)boot_params; mem_avoid[MEM_AVOID_BOOTPARAMS].size = sizeof(*boot_params); - add_identity_map(mem_avoid[MEM_AVOID_BOOTPARAMS].start, - mem_avoid[MEM_AVOID_BOOTPARAMS].size); /* We don't need to set a mapping for setup_data. */ @@ -433,11 +427,6 @@ static void mem_avoid_init(unsigned long input, unsigned long input_size, /* Enumerate the immovable memory regions */ num_immovable_mem = count_immovable_mem_regions(); - -#ifdef CONFIG_X86_VERBOSE_BOOTUP - /* Make sure video RAM can be used. */ - add_identity_map(0, PMD_SIZE); -#endif } /* @@ -884,19 +873,8 @@ void choose_random_location(unsigned long input, warn("Physical KASLR disabled: no suitable memory region!"); } else { /* Update the new physical address location. */ - if (*output != random_addr) { - add_identity_map(random_addr, output_size); + if (*output != random_addr) *output = random_addr; - } - - /* - * This loads the identity mapping page table. - * This should only be done if a new physical address - * is found for the kernel, otherwise we should keep - * the old page table to make it be like the "nokaslr" - * case. - */ - finalize_identity_maps(); } diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index f0e199174c5f..9840c82a39f1 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -98,17 +98,7 @@ static inline void choose_random_location(unsigned long input, #endif #ifdef CONFIG_X86_64 -void initialize_identity_maps(void); -void add_identity_map(unsigned long start, unsigned long size); -void finalize_identity_maps(void); extern unsigned char _pgtable[]; -#else -static inline void initialize_identity_maps(void) -{ } -static inline void add_identity_map(unsigned long start, unsigned long size) -{ } -static inline void finalize_identity_maps(void) -{ } #endif #ifdef CONFIG_EARLY_PRINTK From 21cf2372618ef167d8c4ae04880fb873b55b2daa Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:19 +0200 Subject: [PATCH 031/265] x86/boot/compressed/64: Change add_identity_map() to take start and end Changing the function to take start and end as parameters instead of start and size simplifies the callers which don't need to calculate the size if they already have start and end. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-19-joro@8bytes.org --- arch/x86/boot/compressed/ident_map_64.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index c63257bf8373..62e42c11a336 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -89,10 +89,8 @@ static struct x86_mapping_info mapping_info; /* * Adds the specified range to the identity mappings. */ -static void add_identity_map(unsigned long start, unsigned long size) +static void add_identity_map(unsigned long start, unsigned long end) { - unsigned long end = start + size; - /* Align boundary to 2M. */ start = round_down(start, PMD_SIZE); end = round_up(end, PMD_SIZE); @@ -107,8 +105,6 @@ static void add_identity_map(unsigned long start, unsigned long size) /* Locates and clears a region for a new top level page table. */ void initialize_identity_maps(void) { - unsigned long start, size; - /* If running as an SEV guest, the encryption mask is required. */ set_sev_encryption_mask(); @@ -155,9 +151,7 @@ void initialize_identity_maps(void) * New page-table is set up - map the kernel image and load it * into cr3. */ - start = (unsigned long)_head; - size = _end - _head; - add_identity_map(start, size); + add_identity_map((unsigned long)_head, (unsigned long)_end); write_cr3(top_level_pgt); } @@ -189,7 +183,8 @@ static void do_pf_error(const char *msg, unsigned long error_code, void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code) { - unsigned long address = native_read_cr2(); + unsigned long address = native_read_cr2() & PMD_MASK; + unsigned long end = address + PMD_SIZE; /* * Check for unexpected error codes. Unexpected are: @@ -204,5 +199,5 @@ void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code) * Error code is sane - now identity map the 2M region around * the faulting address. */ - add_identity_map(address & PMD_MASK, PMD_SIZE); + add_identity_map(address, end); } From 29dcc60f6a19fb0aaee97bd1ae2ed8a7dc6f0cfe Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:20 +0200 Subject: [PATCH 032/265] x86/boot/compressed/64: Add stage1 #VC handler Add the first handler for #VC exceptions. At stage 1 there is no GHCB yet because the kernel might still be running on the EFI page table. The stage 1 handler is limited to the MSR-based protocol to talk to the hypervisor and can only support CPUID exit-codes, but that is enough to get to stage 2. [ bp: Zap superfluous newlines after rd/wrmsr instruction mnemonics. ] Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-20-joro@8bytes.org --- arch/x86/boot/compressed/Makefile | 1 + arch/x86/boot/compressed/idt_64.c | 4 ++ arch/x86/boot/compressed/idt_handlers_64.S | 4 ++ arch/x86/boot/compressed/misc.h | 1 + arch/x86/boot/compressed/sev-es.c | 45 +++++++++++++++ arch/x86/include/asm/msr-index.h | 1 + arch/x86/include/asm/sev-es.h | 37 ++++++++++++ arch/x86/include/asm/trapnr.h | 1 + arch/x86/kernel/sev-es-shared.c | 66 ++++++++++++++++++++++ 9 files changed, 160 insertions(+) create mode 100644 arch/x86/boot/compressed/sev-es.c create mode 100644 arch/x86/include/asm/sev-es.h create mode 100644 arch/x86/kernel/sev-es-shared.c diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index e7f3eba99ea2..38f4a52a4eda 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -88,6 +88,7 @@ ifdef CONFIG_X86_64 vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o vmlinux-objs-y += $(obj)/mem_encrypt.o vmlinux-objs-y += $(obj)/pgtable_64.o + vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev-es.o endif vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c index 5f083092a86d..f3ca7324be44 100644 --- a/arch/x86/boot/compressed/idt_64.c +++ b/arch/x86/boot/compressed/idt_64.c @@ -32,6 +32,10 @@ void load_stage1_idt(void) { boot_idt_desc.address = (unsigned long)boot_idt; + + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) + set_idt_entry(X86_TRAP_VC, boot_stage1_vc); + load_boot_idt(&boot_idt_desc); } diff --git a/arch/x86/boot/compressed/idt_handlers_64.S b/arch/x86/boot/compressed/idt_handlers_64.S index b20e57504a94..92eb4df478a1 100644 --- a/arch/x86/boot/compressed/idt_handlers_64.S +++ b/arch/x86/boot/compressed/idt_handlers_64.S @@ -70,3 +70,7 @@ SYM_FUNC_END(\name) .code64 EXCEPTION_HANDLER boot_page_fault do_boot_page_fault error_code=1 + +#ifdef CONFIG_AMD_MEM_ENCRYPT +EXCEPTION_HANDLER boot_stage1_vc do_vc_no_ghcb error_code=1 +#endif diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 9840c82a39f1..eaa8b45ebccb 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -141,5 +141,6 @@ extern struct desc_ptr boot_idt_desc; /* IDT Entry Points */ void boot_page_fault(void); +void boot_stage1_vc(void); #endif /* BOOT_COMPRESSED_MISC_H */ diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c new file mode 100644 index 000000000000..99c3bcd4d61f --- /dev/null +++ b/arch/x86/boot/compressed/sev-es.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD Encrypted Register State Support + * + * Author: Joerg Roedel + */ + +/* + * misc.h needs to be first because it knows how to include the other kernel + * headers in the pre-decompression code in a way that does not break + * compilation. + */ +#include "misc.h" + +#include +#include +#include +#include + +static inline u64 sev_es_rd_ghcb_msr(void) +{ + unsigned long low, high; + + asm volatile("rdmsr" : "=a" (low), "=d" (high) : + "c" (MSR_AMD64_SEV_ES_GHCB)); + + return ((high << 32) | low); +} + +static inline void sev_es_wr_ghcb_msr(u64 val) +{ + u32 low, high; + + low = val & 0xffffffffUL; + high = val >> 32; + + asm volatile("wrmsr" : : "c" (MSR_AMD64_SEV_ES_GHCB), + "a"(low), "d" (high) : "memory"); +} + +#undef __init +#define __init + +/* Include code for early handlers */ +#include "../../kernel/sev-es-shared.c" diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 2859ee4f39a8..da34fdba7c5a 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -466,6 +466,7 @@ #define MSR_AMD64_IBSBRTARGET 0xc001103b #define MSR_AMD64_IBSOPDATA4 0xc001103d #define MSR_AMD64_IBS_REG_COUNT_MAX 8 /* includes MSR_AMD64_IBSBRTARGET */ +#define MSR_AMD64_SEV_ES_GHCB 0xc0010130 #define MSR_AMD64_SEV 0xc0010131 #define MSR_AMD64_SEV_ENABLED_BIT 0 #define MSR_AMD64_SEV_ENABLED BIT_ULL(MSR_AMD64_SEV_ENABLED_BIT) diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h new file mode 100644 index 000000000000..48a44038b5d1 --- /dev/null +++ b/arch/x86/include/asm/sev-es.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AMD Encrypted Register State Support + * + * Author: Joerg Roedel + */ + +#ifndef __ASM_ENCRYPTED_STATE_H +#define __ASM_ENCRYPTED_STATE_H + +#include + +#define GHCB_SEV_CPUID_REQ 0x004UL +#define GHCB_CPUID_REQ_EAX 0 +#define GHCB_CPUID_REQ_EBX 1 +#define GHCB_CPUID_REQ_ECX 2 +#define GHCB_CPUID_REQ_EDX 3 +#define GHCB_CPUID_REQ(fn, reg) (GHCB_SEV_CPUID_REQ | \ + (((unsigned long)reg & 3) << 30) | \ + (((unsigned long)fn) << 32)) + +#define GHCB_SEV_CPUID_RESP 0x005UL +#define GHCB_SEV_TERMINATE 0x100UL + +#define GHCB_SEV_GHCB_RESP_CODE(v) ((v) & 0xfff) +#define VMGEXIT() { asm volatile("rep; vmmcall\n\r"); } + +void do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code); + +static inline u64 lower_bits(u64 val, unsigned int bits) +{ + u64 mask = (1ULL << bits) - 1; + + return (val & mask); +} + +#endif diff --git a/arch/x86/include/asm/trapnr.h b/arch/x86/include/asm/trapnr.h index 082f45631fa9..f5d2325aa0b7 100644 --- a/arch/x86/include/asm/trapnr.h +++ b/arch/x86/include/asm/trapnr.h @@ -26,6 +26,7 @@ #define X86_TRAP_XF 19 /* SIMD Floating-Point Exception */ #define X86_TRAP_VE 20 /* Virtualization Exception */ #define X86_TRAP_CP 21 /* Control Protection Exception */ +#define X86_TRAP_VC 29 /* VMM Communication Exception */ #define X86_TRAP_IRET 32 /* IRET Exception */ #endif diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c new file mode 100644 index 000000000000..0bea32341afa --- /dev/null +++ b/arch/x86/kernel/sev-es-shared.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD Encrypted Register State Support + * + * Author: Joerg Roedel + * + * This file is not compiled stand-alone. It contains code shared + * between the pre-decompression boot code and the running Linux kernel + * and is included directly into both code-bases. + */ + +/* + * Boot VC Handler - This is the first VC handler during boot, there is no GHCB + * page yet, so it only supports the MSR based communication with the + * hypervisor and only the CPUID exit-code. + */ +void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) +{ + unsigned int fn = lower_bits(regs->ax, 32); + unsigned long val; + + /* Only CPUID is supported via MSR protocol */ + if (exit_code != SVM_EXIT_CPUID) + goto fail; + + sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX)); + VMGEXIT(); + val = sev_es_rd_ghcb_msr(); + if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP) + goto fail; + regs->ax = val >> 32; + + sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX)); + VMGEXIT(); + val = sev_es_rd_ghcb_msr(); + if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP) + goto fail; + regs->bx = val >> 32; + + sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX)); + VMGEXIT(); + val = sev_es_rd_ghcb_msr(); + if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP) + goto fail; + regs->cx = val >> 32; + + sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX)); + VMGEXIT(); + val = sev_es_rd_ghcb_msr(); + if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP) + goto fail; + regs->dx = val >> 32; + + /* Skip over the CPUID two-byte opcode */ + regs->ip += 2; + + return; + +fail: + sev_es_wr_ghcb_msr(GHCB_SEV_TERMINATE); + VMGEXIT(); + + /* Shouldn't get here - if we do halt the machine */ + while (true) + asm volatile("hlt\n"); +} From c2a0304a286f386e45cea3f4b0617f0813de67fd Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:21 +0200 Subject: [PATCH 033/265] x86/boot/compressed/64: Call set_sev_encryption_mask() earlier Call set_sev_encryption_mask() while still on the stage 1 #VC-handler because the stage 2 handler needs the kernel's own page tables to be set up, to which calling set_sev_encryption_mask() is a prerequisite. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-21-joro@8bytes.org --- arch/x86/boot/compressed/head_64.S | 9 ++++++++- arch/x86/boot/compressed/ident_map_64.c | 3 --- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index fb6c0392306b..42190c00d9c2 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -543,9 +543,16 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated) rep stosq /* - * Load stage2 IDT and switch to our own page-table + * If running as an SEV guest, the encryption mask is required in the + * page-table setup code below. When the guest also has SEV-ES enabled + * set_sev_encryption_mask() will cause #VC exceptions, but the stage2 + * handler can't map its GHCB because the page-table is not set up yet. + * So set up the encryption mask here while still on the stage1 #VC + * handler. Then load stage2 IDT and switch to the kernel's own + * page-table. */ pushq %rsi + call set_sev_encryption_mask call load_stage2_idt call initialize_identity_maps popq %rsi diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 62e42c11a336..b4f2a5f503cd 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -105,9 +105,6 @@ static void add_identity_map(unsigned long start, unsigned long end) /* Locates and clears a region for a new top level page table. */ void initialize_identity_maps(void) { - /* If running as an SEV guest, the encryption mask is required. */ - set_sev_encryption_mask(); - /* Exclude the encryption mask from __PHYSICAL_MASK */ physical_mask &= ~sme_me_mask; From 4b3fdca64a7e8ad90c87cad1fbc6991471f48dc7 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:22 +0200 Subject: [PATCH 034/265] x86/boot/compressed/64: Check return value of kernel_ident_mapping_init() The function can fail to create an identity mapping, check for that and bail out if it happens. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-22-joro@8bytes.org --- arch/x86/boot/compressed/ident_map_64.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index b4f2a5f503cd..aa91bebc0fe9 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -91,6 +91,8 @@ static struct x86_mapping_info mapping_info; */ static void add_identity_map(unsigned long start, unsigned long end) { + int ret; + /* Align boundary to 2M. */ start = round_down(start, PMD_SIZE); end = round_up(end, PMD_SIZE); @@ -98,8 +100,9 @@ static void add_identity_map(unsigned long start, unsigned long end) return; /* Build the mapping. */ - kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, - start, end); + ret = kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, start, end); + if (ret) + error("Error: kernel_ident_mapping_init() failed\n"); } /* Locates and clears a region for a new top level page table. */ From c81d60029a1393183d2125fcb4b64831629b8864 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:23 +0200 Subject: [PATCH 035/265] x86/boot/compressed/64: Add set_page_en/decrypted() helpers The functions are needed to map the GHCB for SEV-ES guests. The GHCB is used for communication with the hypervisor, so its content must not be encrypted. After the GHCB is not needed anymore it must be mapped encrypted again so that the running kernel image can safely re-use the memory. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-23-joro@8bytes.org --- arch/x86/boot/compressed/ident_map_64.c | 133 ++++++++++++++++++++++++ arch/x86/boot/compressed/misc.h | 2 + 2 files changed, 135 insertions(+) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index aa91bebc0fe9..05742f641a06 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -24,6 +24,7 @@ /* These actually do the work of building the kernel identity maps. */ #include +#include #include #include #include @@ -165,6 +166,138 @@ void finalize_identity_maps(void) write_cr3(top_level_pgt); } +static pte_t *split_large_pmd(struct x86_mapping_info *info, + pmd_t *pmdp, unsigned long __address) +{ + unsigned long page_flags; + unsigned long address; + pte_t *pte; + pmd_t pmd; + int i; + + pte = (pte_t *)info->alloc_pgt_page(info->context); + if (!pte) + return NULL; + + address = __address & PMD_MASK; + /* No large page - clear PSE flag */ + page_flags = info->page_flag & ~_PAGE_PSE; + + /* Populate the PTEs */ + for (i = 0; i < PTRS_PER_PMD; i++) { + set_pte(&pte[i], __pte(address | page_flags)); + address += PAGE_SIZE; + } + + /* + * Ideally we need to clear the large PMD first and do a TLB + * flush before we write the new PMD. But the 2M range of the + * PMD might contain the code we execute and/or the stack + * we are on, so we can't do that. But that should be safe here + * because we are going from large to small mappings and we are + * also the only user of the page-table, so there is no chance + * of a TLB multihit. + */ + pmd = __pmd((unsigned long)pte | info->kernpg_flag); + set_pmd(pmdp, pmd); + /* Flush TLB to establish the new PMD */ + write_cr3(top_level_pgt); + + return pte + pte_index(__address); +} + +static void clflush_page(unsigned long address) +{ + unsigned int flush_size; + char *cl, *start, *end; + + /* + * Hardcode cl-size to 64 - CPUID can't be used here because that might + * cause another #VC exception and the GHCB is not ready to use yet. + */ + flush_size = 64; + start = (char *)(address & PAGE_MASK); + end = start + PAGE_SIZE; + + /* + * First make sure there are no pending writes on the cache-lines to + * flush. + */ + asm volatile("mfence" : : : "memory"); + + for (cl = start; cl != end; cl += flush_size) + clflush(cl); +} + +static int set_clr_page_flags(struct x86_mapping_info *info, + unsigned long address, + pteval_t set, pteval_t clr) +{ + pgd_t *pgdp = (pgd_t *)top_level_pgt; + p4d_t *p4dp; + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep, pte; + + /* + * First make sure there is a PMD mapping for 'address'. + * It should already exist, but keep things generic. + * + * To map the page just read from it and fault it in if there is no + * mapping yet. add_identity_map() can't be called here because that + * would unconditionally map the address on PMD level, destroying any + * PTE-level mappings that might already exist. Use assembly here so + * the access won't be optimized away. + */ + asm volatile("mov %[address], %%r9" + :: [address] "g" (*(unsigned long *)address) + : "r9", "memory"); + + /* + * The page is mapped at least with PMD size - so skip checks and walk + * directly to the PMD. + */ + p4dp = p4d_offset(pgdp, address); + pudp = pud_offset(p4dp, address); + pmdp = pmd_offset(pudp, address); + + if (pmd_large(*pmdp)) + ptep = split_large_pmd(info, pmdp, address); + else + ptep = pte_offset_kernel(pmdp, address); + + if (!ptep) + return -ENOMEM; + + /* + * Changing encryption attributes of a page requires to flush it from + * the caches. + */ + if ((set | clr) & _PAGE_ENC) + clflush_page(address); + + /* Update PTE */ + pte = *ptep; + pte = pte_set_flags(pte, set); + pte = pte_clear_flags(pte, clr); + set_pte(ptep, pte); + + /* Flush TLB after changing encryption attribute */ + write_cr3(top_level_pgt); + + return 0; +} + +int set_page_decrypted(unsigned long address) +{ + return set_clr_page_flags(&mapping_info, address, 0, _PAGE_ENC); +} + +int set_page_encrypted(unsigned long address) +{ + return set_clr_page_flags(&mapping_info, address, _PAGE_ENC, 0); +} + static void do_pf_error(const char *msg, unsigned long error_code, unsigned long address, unsigned long ip) { diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index eaa8b45ebccb..01c0fb3417ca 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -98,6 +98,8 @@ static inline void choose_random_location(unsigned long input, #endif #ifdef CONFIG_X86_64 +extern int set_page_decrypted(unsigned long address); +extern int set_page_encrypted(unsigned long address); extern unsigned char _pgtable[]; #endif From 597cfe48212a3f110ab0f918bf59791f453e65b7 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:24 +0200 Subject: [PATCH 036/265] x86/boot/compressed/64: Setup a GHCB-based VC Exception handler Install an exception handler for #VC exception that uses a GHCB. Also add the infrastructure for handling different exit-codes by decoding the instruction that caused the exception and error handling. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-24-joro@8bytes.org --- arch/x86/Kconfig | 1 + arch/x86/boot/compressed/Makefile | 5 + arch/x86/boot/compressed/idt_64.c | 4 + arch/x86/boot/compressed/idt_handlers_64.S | 3 +- arch/x86/boot/compressed/misc.c | 7 + arch/x86/boot/compressed/misc.h | 7 + arch/x86/boot/compressed/sev-es.c | 111 +++++++++++++++ arch/x86/include/asm/sev-es.h | 39 ++++++ arch/x86/include/uapi/asm/svm.h | 1 + arch/x86/kernel/sev-es-shared.c | 154 +++++++++++++++++++++ 10 files changed, 331 insertions(+), 1 deletion(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 7101ac64bb20..8289dd44efbd 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1521,6 +1521,7 @@ config AMD_MEM_ENCRYPT select DYNAMIC_PHYSICAL_MASK select ARCH_USE_MEMREMAP_PROT select ARCH_HAS_FORCE_DMA_UNENCRYPTED + select INSTRUCTION_DECODER help Say yes to enable support for the encryption of system memory. This requires an AMD processor that supports Secure Memory diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 38f4a52a4eda..c01236ae1b7c 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -44,6 +44,11 @@ KBUILD_CFLAGS += $(call cc-option,-fmacro-prefix-map=$(srctree)/=) KBUILD_CFLAGS += -fno-asynchronous-unwind-tables KBUILD_CFLAGS += -D__DISABLE_EXPORTS +# sev-es.c indirectly inludes inat-table.h which is generated during +# compilation and stored in $(objtree). Add the directory to the includes so +# that the compiler finds it even with out-of-tree builds (make O=/some/path). +CFLAGS_sev-es.o += -I$(objtree)/arch/x86/lib/ + KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ GCOV_PROFILE := n UBSAN_SANITIZE :=n diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c index f3ca7324be44..804a502ee0d2 100644 --- a/arch/x86/boot/compressed/idt_64.c +++ b/arch/x86/boot/compressed/idt_64.c @@ -46,5 +46,9 @@ void load_stage2_idt(void) set_idt_entry(X86_TRAP_PF, boot_page_fault); +#ifdef CONFIG_AMD_MEM_ENCRYPT + set_idt_entry(X86_TRAP_VC, boot_stage2_vc); +#endif + load_boot_idt(&boot_idt_desc); } diff --git a/arch/x86/boot/compressed/idt_handlers_64.S b/arch/x86/boot/compressed/idt_handlers_64.S index 92eb4df478a1..22890e199f5b 100644 --- a/arch/x86/boot/compressed/idt_handlers_64.S +++ b/arch/x86/boot/compressed/idt_handlers_64.S @@ -72,5 +72,6 @@ SYM_FUNC_END(\name) EXCEPTION_HANDLER boot_page_fault do_boot_page_fault error_code=1 #ifdef CONFIG_AMD_MEM_ENCRYPT -EXCEPTION_HANDLER boot_stage1_vc do_vc_no_ghcb error_code=1 +EXCEPTION_HANDLER boot_stage1_vc do_vc_no_ghcb error_code=1 +EXCEPTION_HANDLER boot_stage2_vc do_boot_stage2_vc error_code=1 #endif diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index e478e40fbe5a..267e7f93050e 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -442,6 +442,13 @@ asmlinkage __visible void *extract_kernel(void *rmode, memptr heap, parse_elf(output); handle_relocations(output, output_len, virt_addr); debug_putstr("done.\nBooting the kernel.\n"); + + /* + * Flush GHCB from cache and map it encrypted again when running as + * SEV-ES guest. + */ + sev_es_shutdown_ghcb(); + return output; } diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 01c0fb3417ca..9995c70ca813 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -115,6 +115,12 @@ static inline void console_init(void) void set_sev_encryption_mask(void); +#ifdef CONFIG_AMD_MEM_ENCRYPT +void sev_es_shutdown_ghcb(void); +#else +static inline void sev_es_shutdown_ghcb(void) { } +#endif + /* acpi.c */ #ifdef CONFIG_ACPI acpi_physical_address get_rsdp_addr(void); @@ -144,5 +150,6 @@ extern struct desc_ptr boot_idt_desc; /* IDT Entry Points */ void boot_page_fault(void); void boot_stage1_vc(void); +void boot_stage2_vc(void); #endif /* BOOT_COMPRESSED_MISC_H */ diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index 99c3bcd4d61f..fa62af771dd5 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -13,10 +13,17 @@ #include "misc.h" #include +#include +#include #include #include #include +#include "error.h" + +struct ghcb boot_ghcb_page __aligned(PAGE_SIZE); +struct ghcb *boot_ghcb; + static inline u64 sev_es_rd_ghcb_msr(void) { unsigned long low, high; @@ -38,8 +45,112 @@ static inline void sev_es_wr_ghcb_msr(u64 val) "a"(low), "d" (high) : "memory"); } +static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) +{ + char buffer[MAX_INSN_SIZE]; + enum es_result ret; + + memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); + + insn_init(&ctxt->insn, buffer, MAX_INSN_SIZE, 1); + insn_get_length(&ctxt->insn); + + ret = ctxt->insn.immediate.got ? ES_OK : ES_DECODE_FAILED; + + return ret; +} + +static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, + void *dst, char *buf, size_t size) +{ + memcpy(dst, buf, size); + + return ES_OK; +} + +static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, + void *src, char *buf, size_t size) +{ + memcpy(buf, src, size); + + return ES_OK; +} + #undef __init +#undef __pa #define __init +#define __pa(x) ((unsigned long)(x)) + +#define __BOOT_COMPRESSED + +/* Basic instruction decoding support needed */ +#include "../../lib/inat.c" +#include "../../lib/insn.c" /* Include code for early handlers */ #include "../../kernel/sev-es-shared.c" + +static bool early_setup_sev_es(void) +{ + if (!sev_es_negotiate_protocol()) + sev_es_terminate(GHCB_SEV_ES_REASON_PROTOCOL_UNSUPPORTED); + + if (set_page_decrypted((unsigned long)&boot_ghcb_page)) + return false; + + /* Page is now mapped decrypted, clear it */ + memset(&boot_ghcb_page, 0, sizeof(boot_ghcb_page)); + + boot_ghcb = &boot_ghcb_page; + + /* Initialize lookup tables for the instruction decoder */ + inat_init_tables(); + + return true; +} + +void sev_es_shutdown_ghcb(void) +{ + if (!boot_ghcb) + return; + + /* + * GHCB Page must be flushed from the cache and mapped encrypted again. + * Otherwise the running kernel will see strange cache effects when + * trying to use that page. + */ + if (set_page_encrypted((unsigned long)&boot_ghcb_page)) + error("Can't map GHCB page encrypted"); +} + +void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) +{ + struct es_em_ctxt ctxt; + enum es_result result; + + if (!boot_ghcb && !early_setup_sev_es()) + sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); + + vc_ghcb_invalidate(boot_ghcb); + result = vc_init_em_ctxt(&ctxt, regs, exit_code); + if (result != ES_OK) + goto finish; + + switch (exit_code) { + default: + result = ES_UNSUPPORTED; + break; + } + +finish: + if (result == ES_OK) { + vc_finish_insn(&ctxt); + } else if (result != ES_RETRY) { + /* + * For now, just halt the machine. That makes debugging easier, + * later we just call sev_es_terminate() here. + */ + while (true) + asm volatile("hlt\n"); + } +} diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index 48a44038b5d1..6dc52440c4b4 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -9,7 +9,14 @@ #define __ASM_ENCRYPTED_STATE_H #include +#include +#define GHCB_SEV_INFO 0x001UL +#define GHCB_SEV_INFO_REQ 0x002UL +#define GHCB_INFO(v) ((v) & 0xfffUL) +#define GHCB_PROTO_MAX(v) (((v) >> 48) & 0xffffUL) +#define GHCB_PROTO_MIN(v) (((v) >> 32) & 0xffffUL) +#define GHCB_PROTO_OUR 0x0001UL #define GHCB_SEV_CPUID_REQ 0x004UL #define GHCB_CPUID_REQ_EAX 0 #define GHCB_CPUID_REQ_EBX 1 @@ -19,12 +26,44 @@ (((unsigned long)reg & 3) << 30) | \ (((unsigned long)fn) << 32)) +#define GHCB_PROTOCOL_MAX 0x0001UL +#define GHCB_DEFAULT_USAGE 0x0000UL + #define GHCB_SEV_CPUID_RESP 0x005UL #define GHCB_SEV_TERMINATE 0x100UL +#define GHCB_SEV_TERMINATE_REASON(reason_set, reason_val) \ + (((((u64)reason_set) & 0x7) << 12) | \ + ((((u64)reason_val) & 0xff) << 16)) +#define GHCB_SEV_ES_REASON_GENERAL_REQUEST 0 +#define GHCB_SEV_ES_REASON_PROTOCOL_UNSUPPORTED 1 #define GHCB_SEV_GHCB_RESP_CODE(v) ((v) & 0xfff) #define VMGEXIT() { asm volatile("rep; vmmcall\n\r"); } +enum es_result { + ES_OK, /* All good */ + ES_UNSUPPORTED, /* Requested operation not supported */ + ES_VMM_ERROR, /* Unexpected state from the VMM */ + ES_DECODE_FAILED, /* Instruction decoding failed */ + ES_EXCEPTION, /* Instruction caused exception */ + ES_RETRY, /* Retry instruction emulation */ +}; + +struct es_fault_info { + unsigned long vector; + unsigned long error_code; + unsigned long cr2; +}; + +struct pt_regs; + +/* ES instruction emulation context */ +struct es_em_ctxt { + struct pt_regs *regs; + struct insn insn; + struct es_fault_info fi; +}; + void do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code); static inline u64 lower_bits(u64 val, unsigned int bits) diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index 2e8a30f06c74..c68d1618c9b0 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -29,6 +29,7 @@ #define SVM_EXIT_WRITE_DR6 0x036 #define SVM_EXIT_WRITE_DR7 0x037 #define SVM_EXIT_EXCP_BASE 0x040 +#define SVM_EXIT_LAST_EXCP 0x05f #define SVM_EXIT_INTR 0x060 #define SVM_EXIT_NMI 0x061 #define SVM_EXIT_SMI 0x062 diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 0bea32341afa..7ac6e6b0ae57 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -9,6 +9,118 @@ * and is included directly into both code-bases. */ +static void sev_es_terminate(unsigned int reason) +{ + u64 val = GHCB_SEV_TERMINATE; + + /* + * Tell the hypervisor what went wrong - only reason-set 0 is + * currently supported. + */ + val |= GHCB_SEV_TERMINATE_REASON(0, reason); + + /* Request Guest Termination from Hypvervisor */ + sev_es_wr_ghcb_msr(val); + VMGEXIT(); + + while (true) + asm volatile("hlt\n" : : : "memory"); +} + +static bool sev_es_negotiate_protocol(void) +{ + u64 val; + + /* Do the GHCB protocol version negotiation */ + sev_es_wr_ghcb_msr(GHCB_SEV_INFO_REQ); + VMGEXIT(); + val = sev_es_rd_ghcb_msr(); + + if (GHCB_INFO(val) != GHCB_SEV_INFO) + return false; + + if (GHCB_PROTO_MAX(val) < GHCB_PROTO_OUR || + GHCB_PROTO_MIN(val) > GHCB_PROTO_OUR) + return false; + + return true; +} + +static void vc_ghcb_invalidate(struct ghcb *ghcb) +{ + memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); +} + +static bool vc_decoding_needed(unsigned long exit_code) +{ + /* Exceptions don't require to decode the instruction */ + return !(exit_code >= SVM_EXIT_EXCP_BASE && + exit_code <= SVM_EXIT_LAST_EXCP); +} + +static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, + struct pt_regs *regs, + unsigned long exit_code) +{ + enum es_result ret = ES_OK; + + memset(ctxt, 0, sizeof(*ctxt)); + ctxt->regs = regs; + + if (vc_decoding_needed(exit_code)) + ret = vc_decode_insn(ctxt); + + return ret; +} + +static void vc_finish_insn(struct es_em_ctxt *ctxt) +{ + ctxt->regs->ip += ctxt->insn.length; +} + +static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, + struct es_em_ctxt *ctxt, + u64 exit_code, u64 exit_info_1, + u64 exit_info_2) +{ + enum es_result ret; + + /* Fill in protocol and format specifiers */ + ghcb->protocol_version = GHCB_PROTOCOL_MAX; + ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; + + ghcb_set_sw_exit_code(ghcb, exit_code); + ghcb_set_sw_exit_info_1(ghcb, exit_info_1); + ghcb_set_sw_exit_info_2(ghcb, exit_info_2); + + sev_es_wr_ghcb_msr(__pa(ghcb)); + VMGEXIT(); + + if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) { + u64 info = ghcb->save.sw_exit_info_2; + unsigned long v; + + info = ghcb->save.sw_exit_info_2; + v = info & SVM_EVTINJ_VEC_MASK; + + /* Check if exception information from hypervisor is sane. */ + if ((info & SVM_EVTINJ_VALID) && + ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && + ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { + ctxt->fi.vector = v; + if (info & SVM_EVTINJ_VALID_ERR) + ctxt->fi.error_code = info >> 32; + ret = ES_EXCEPTION; + } else { + ret = ES_VMM_ERROR; + } + } else { + ret = ES_OK; + } + + return ret; +} + /* * Boot VC Handler - This is the first VC handler during boot, there is no GHCB * page yet, so it only supports the MSR based communication with the @@ -64,3 +176,45 @@ fail: while (true) asm volatile("hlt\n"); } + +static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, + void *src, char *buf, + unsigned int data_size, + unsigned int count, + bool backwards) +{ + int i, b = backwards ? -1 : 1; + enum es_result ret = ES_OK; + + for (i = 0; i < count; i++) { + void *s = src + (i * data_size * b); + char *d = buf + (i * data_size); + + ret = vc_read_mem(ctxt, s, d, data_size); + if (ret != ES_OK) + break; + } + + return ret; +} + +static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, + void *dst, char *buf, + unsigned int data_size, + unsigned int count, + bool backwards) +{ + int i, s = backwards ? -1 : 1; + enum es_result ret = ES_OK; + + for (i = 0; i < count; i++) { + void *d = dst + (i * data_size * s); + char *b = buf + (i * data_size); + + ret = vc_write_mem(ctxt, d, b, data_size); + if (ret != ES_OK) + break; + } + + return ret; +} From 69add17a7c1992593a7cf775a66e0256ad4b3ef8 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:25 +0200 Subject: [PATCH 037/265] x86/boot/compressed/64: Unmap GHCB page before booting the kernel Force a page-fault on any further accesses to the GHCB page when they shouldn't happen anymore. This will catch any bugs where a #VC exception is raised even though none is expected anymore. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-25-joro@8bytes.org --- arch/x86/boot/compressed/ident_map_64.c | 17 +++++++++++++++-- arch/x86/boot/compressed/misc.h | 6 ++++++ arch/x86/boot/compressed/sev-es.c | 14 ++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 05742f641a06..063a60edcf99 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -298,6 +298,11 @@ int set_page_encrypted(unsigned long address) return set_clr_page_flags(&mapping_info, address, _PAGE_ENC, 0); } +int set_page_non_present(unsigned long address) +{ + return set_clr_page_flags(&mapping_info, address, 0, _PAGE_PRESENT); +} + static void do_pf_error(const char *msg, unsigned long error_code, unsigned long address, unsigned long ip) { @@ -316,8 +321,14 @@ static void do_pf_error(const char *msg, unsigned long error_code, void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code) { - unsigned long address = native_read_cr2() & PMD_MASK; - unsigned long end = address + PMD_SIZE; + unsigned long address = native_read_cr2(); + unsigned long end; + bool ghcb_fault; + + ghcb_fault = sev_es_check_ghcb_fault(address); + + address &= PMD_MASK; + end = address + PMD_SIZE; /* * Check for unexpected error codes. Unexpected are: @@ -327,6 +338,8 @@ void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code) */ if (error_code & (X86_PF_PROT | X86_PF_USER | X86_PF_RSVD)) do_pf_error("Unexpected page-fault:", error_code, address, regs->ip); + else if (ghcb_fault) + do_pf_error("Page-fault on GHCB page:", error_code, address, regs->ip); /* * Error code is sane - now identity map the 2M region around diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 9995c70ca813..c0e0ffeee50a 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -100,6 +100,7 @@ static inline void choose_random_location(unsigned long input, #ifdef CONFIG_X86_64 extern int set_page_decrypted(unsigned long address); extern int set_page_encrypted(unsigned long address); +extern int set_page_non_present(unsigned long address); extern unsigned char _pgtable[]; #endif @@ -117,8 +118,13 @@ void set_sev_encryption_mask(void); #ifdef CONFIG_AMD_MEM_ENCRYPT void sev_es_shutdown_ghcb(void); +extern bool sev_es_check_ghcb_fault(unsigned long address); #else static inline void sev_es_shutdown_ghcb(void) { } +static inline bool sev_es_check_ghcb_fault(unsigned long address) +{ + return false; +} #endif /* acpi.c */ diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index fa62af771dd5..1e1fab583302 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -121,6 +121,20 @@ void sev_es_shutdown_ghcb(void) */ if (set_page_encrypted((unsigned long)&boot_ghcb_page)) error("Can't map GHCB page encrypted"); + + /* + * GHCB page is mapped encrypted again and flushed from the cache. + * Mark it non-present now to catch bugs when #VC exceptions trigger + * after this point. + */ + if (set_page_non_present((unsigned long)&boot_ghcb_page)) + error("Can't unmap GHCB page"); +} + +bool sev_es_check_ghcb_fault(unsigned long address) +{ + /* Check whether the fault was on the GHCB page */ + return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page); } void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) From 25189d08e5168c098c307a0eaae5b30c13a331ef Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:26 +0200 Subject: [PATCH 038/265] x86/sev-es: Add support for handling IOIO exceptions Add support for decoding and handling #VC exceptions for IOIO events. [ jroedel@suse.de: Adapted code to #VC handling framework ] Co-developed-by: Joerg Roedel Signed-off-by: Tom Lendacky Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-26-joro@8bytes.org --- arch/x86/boot/compressed/sev-es.c | 32 +++++ arch/x86/kernel/sev-es-shared.c | 214 ++++++++++++++++++++++++++++++ 2 files changed, 246 insertions(+) diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index 1e1fab583302..61504eb1ab46 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -24,6 +24,35 @@ struct ghcb boot_ghcb_page __aligned(PAGE_SIZE); struct ghcb *boot_ghcb; +/* + * Copy a version of this function here - insn-eval.c can't be used in + * pre-decompression code. + */ +static bool insn_has_rep_prefix(struct insn *insn) +{ + int i; + + insn_get_prefixes(insn); + + for (i = 0; i < insn->prefixes.nbytes; i++) { + insn_byte_t p = insn->prefixes.bytes[i]; + + if (p == 0xf2 || p == 0xf3) + return true; + } + + return false; +} + +/* + * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and + * doesn't use segments. + */ +static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx) +{ + return 0UL; +} + static inline u64 sev_es_rd_ghcb_msr(void) { unsigned long low, high; @@ -151,6 +180,9 @@ void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) goto finish; switch (exit_code) { + case SVM_EXIT_IOIO: + result = vc_handle_ioio(boot_ghcb, &ctxt); + break; default: result = ES_UNSUPPORTED; break; diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 7ac6e6b0ae57..bae7cf28455b 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -218,3 +218,217 @@ static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, return ret; } + +#define IOIO_TYPE_STR BIT(2) +#define IOIO_TYPE_IN 1 +#define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) +#define IOIO_TYPE_OUT 0 +#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) + +#define IOIO_REP BIT(3) + +#define IOIO_ADDR_64 BIT(9) +#define IOIO_ADDR_32 BIT(8) +#define IOIO_ADDR_16 BIT(7) + +#define IOIO_DATA_32 BIT(6) +#define IOIO_DATA_16 BIT(5) +#define IOIO_DATA_8 BIT(4) + +#define IOIO_SEG_ES (0 << 10) +#define IOIO_SEG_DS (3 << 10) + +static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) +{ + struct insn *insn = &ctxt->insn; + *exitinfo = 0; + + switch (insn->opcode.bytes[0]) { + /* INS opcodes */ + case 0x6c: + case 0x6d: + *exitinfo |= IOIO_TYPE_INS; + *exitinfo |= IOIO_SEG_ES; + *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; + break; + + /* OUTS opcodes */ + case 0x6e: + case 0x6f: + *exitinfo |= IOIO_TYPE_OUTS; + *exitinfo |= IOIO_SEG_DS; + *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; + break; + + /* IN immediate opcodes */ + case 0xe4: + case 0xe5: + *exitinfo |= IOIO_TYPE_IN; + *exitinfo |= (u64)insn->immediate.value << 16; + break; + + /* OUT immediate opcodes */ + case 0xe6: + case 0xe7: + *exitinfo |= IOIO_TYPE_OUT; + *exitinfo |= (u64)insn->immediate.value << 16; + break; + + /* IN register opcodes */ + case 0xec: + case 0xed: + *exitinfo |= IOIO_TYPE_IN; + *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; + break; + + /* OUT register opcodes */ + case 0xee: + case 0xef: + *exitinfo |= IOIO_TYPE_OUT; + *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; + break; + + default: + return ES_DECODE_FAILED; + } + + switch (insn->opcode.bytes[0]) { + case 0x6c: + case 0x6e: + case 0xe4: + case 0xe6: + case 0xec: + case 0xee: + /* Single byte opcodes */ + *exitinfo |= IOIO_DATA_8; + break; + default: + /* Length determined by instruction parsing */ + *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 + : IOIO_DATA_32; + } + switch (insn->addr_bytes) { + case 2: + *exitinfo |= IOIO_ADDR_16; + break; + case 4: + *exitinfo |= IOIO_ADDR_32; + break; + case 8: + *exitinfo |= IOIO_ADDR_64; + break; + } + + if (insn_has_rep_prefix(insn)) + *exitinfo |= IOIO_REP; + + return ES_OK; +} + +static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + struct pt_regs *regs = ctxt->regs; + u64 exit_info_1, exit_info_2; + enum es_result ret; + + ret = vc_ioio_exitinfo(ctxt, &exit_info_1); + if (ret != ES_OK) + return ret; + + if (exit_info_1 & IOIO_TYPE_STR) { + + /* (REP) INS/OUTS */ + + bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); + unsigned int io_bytes, exit_bytes; + unsigned int ghcb_count, op_count; + unsigned long es_base; + u64 sw_scratch; + + /* + * For the string variants with rep prefix the amount of in/out + * operations per #VC exception is limited so that the kernel + * has a chance to take interrupts and re-schedule while the + * instruction is emulated. + */ + io_bytes = (exit_info_1 >> 4) & 0x7; + ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; + + op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; + exit_info_2 = min(op_count, ghcb_count); + exit_bytes = exit_info_2 * io_bytes; + + es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); + + /* Read bytes of OUTS into the shared buffer */ + if (!(exit_info_1 & IOIO_TYPE_IN)) { + ret = vc_insn_string_read(ctxt, + (void *)(es_base + regs->si), + ghcb->shared_buffer, io_bytes, + exit_info_2, df); + if (ret) + return ret; + } + + /* + * Issue an VMGEXIT to the HV to consume the bytes from the + * shared buffer or to have it write them into the shared buffer + * depending on the instruction: OUTS or INS. + */ + sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); + ghcb_set_sw_scratch(ghcb, sw_scratch); + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, + exit_info_1, exit_info_2); + if (ret != ES_OK) + return ret; + + /* Read bytes from shared buffer into the guest's destination. */ + if (exit_info_1 & IOIO_TYPE_IN) { + ret = vc_insn_string_write(ctxt, + (void *)(es_base + regs->di), + ghcb->shared_buffer, io_bytes, + exit_info_2, df); + if (ret) + return ret; + + if (df) + regs->di -= exit_bytes; + else + regs->di += exit_bytes; + } else { + if (df) + regs->si -= exit_bytes; + else + regs->si += exit_bytes; + } + + if (exit_info_1 & IOIO_REP) + regs->cx -= exit_info_2; + + ret = regs->cx ? ES_RETRY : ES_OK; + + } else { + + /* IN/OUT into/from rAX */ + + int bits = (exit_info_1 & 0x70) >> 1; + u64 rax = 0; + + if (!(exit_info_1 & IOIO_TYPE_IN)) + rax = lower_bits(regs->ax, bits); + + ghcb_set_rax(ghcb, rax); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); + if (ret != ES_OK) + return ret; + + if (exit_info_1 & IOIO_TYPE_IN) { + if (!ghcb_rax_is_valid(ghcb)) + return ES_VMM_ERROR; + regs->ax = lower_bits(ghcb->save.rax, bits); + } + } + + return ret; +} From 1b4fb8545f2b00f2844c4b7619d64d98440a477c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:27 +0200 Subject: [PATCH 039/265] x86/fpu: Move xgetbv()/xsetbv() into a separate header The xgetbv() function is needed in the pre-decompression boot code, but asm/fpu/internal.h can't be included there directly. Doing so opens the door to include-hell due to various include-magic in boot/compressed/misc.h. Avoid that by moving xgetbv()/xsetbv() to a separate header file and include it instead. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-27-joro@8bytes.org --- arch/x86/include/asm/fpu/internal.h | 30 +------------------------ arch/x86/include/asm/fpu/xcr.h | 34 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 29 deletions(-) create mode 100644 arch/x86/include/asm/fpu/xcr.h diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h index 21a8b5259477..ceeba9f63172 100644 --- a/arch/x86/include/asm/fpu/internal.h +++ b/arch/x86/include/asm/fpu/internal.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -585,33 +586,4 @@ static inline void switch_fpu_finish(struct fpu *new_fpu) __write_pkru(pkru_val); } -/* - * MXCSR and XCR definitions: - */ - -static inline void ldmxcsr(u32 mxcsr) -{ - asm volatile("ldmxcsr %0" :: "m" (mxcsr)); -} - -extern unsigned int mxcsr_feature_mask; - -#define XCR_XFEATURE_ENABLED_MASK 0x00000000 - -static inline u64 xgetbv(u32 index) -{ - u32 eax, edx; - - asm volatile("xgetbv" : "=a" (eax), "=d" (edx) : "c" (index)); - return eax + ((u64)edx << 32); -} - -static inline void xsetbv(u32 index, u64 value) -{ - u32 eax = value; - u32 edx = value >> 32; - - asm volatile("xsetbv" :: "a" (eax), "d" (edx), "c" (index)); -} - #endif /* _ASM_X86_FPU_INTERNAL_H */ diff --git a/arch/x86/include/asm/fpu/xcr.h b/arch/x86/include/asm/fpu/xcr.h new file mode 100644 index 000000000000..1c7ab8d95da5 --- /dev/null +++ b/arch/x86/include/asm/fpu/xcr.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_FPU_XCR_H +#define _ASM_X86_FPU_XCR_H + +/* + * MXCSR and XCR definitions: + */ + +static inline void ldmxcsr(u32 mxcsr) +{ + asm volatile("ldmxcsr %0" :: "m" (mxcsr)); +} + +extern unsigned int mxcsr_feature_mask; + +#define XCR_XFEATURE_ENABLED_MASK 0x00000000 + +static inline u64 xgetbv(u32 index) +{ + u32 eax, edx; + + asm volatile("xgetbv" : "=a" (eax), "=d" (edx) : "c" (index)); + return eax + ((u64)edx << 32); +} + +static inline void xsetbv(u32 index, u64 value) +{ + u32 eax = value; + u32 edx = value >> 32; + + asm volatile("xsetbv" :: "a" (eax), "d" (edx), "c" (index)); +} + +#endif /* _ASM_X86_FPU_XCR_H */ From a7de15d489d956217b47671705ac2218ca50eaae Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:28 +0200 Subject: [PATCH 040/265] x86/sev-es: Add CPUID handling to #VC handler Handle #VC exceptions caused by CPUID instructions. These happen in early boot code when the KASLR code checks for RDTSC. Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling framework ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-28-joro@8bytes.org --- arch/x86/boot/compressed/sev-es.c | 4 ++++ arch/x86/kernel/sev-es-shared.c | 35 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index 61504eb1ab46..b1790f487456 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -183,6 +184,9 @@ void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) case SVM_EXIT_IOIO: result = vc_handle_ioio(boot_ghcb, &ctxt); break; + case SVM_EXIT_CPUID: + result = vc_handle_cpuid(boot_ghcb, &ctxt); + break; default: result = ES_UNSUPPORTED; break; diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index bae7cf28455b..a6b41910b8ab 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -432,3 +432,38 @@ static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) return ret; } + +static enum es_result vc_handle_cpuid(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct pt_regs *regs = ctxt->regs; + u32 cr4 = native_read_cr4(); + enum es_result ret; + + ghcb_set_rax(ghcb, regs->ax); + ghcb_set_rcx(ghcb, regs->cx); + + if (cr4 & X86_CR4_OSXSAVE) + /* Safe to read xcr0 */ + ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); + else + /* xgetbv will cause #GP - use reset value for xcr0 */ + ghcb_set_xcr0(ghcb, 1); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); + if (ret != ES_OK) + return ret; + + if (!(ghcb_rax_is_valid(ghcb) && + ghcb_rbx_is_valid(ghcb) && + ghcb_rcx_is_valid(ghcb) && + ghcb_rdx_is_valid(ghcb))) + return ES_VMM_ERROR; + + regs->ax = ghcb->save.rax; + regs->bx = ghcb->save.rbx; + regs->cx = ghcb->save.rcx; + regs->dx = ghcb->save.rdx; + + return ES_OK; +} From 4bed2266cc6f9c3f6cd91378ea4fc76edde674cf Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:29 +0200 Subject: [PATCH 041/265] x86/idt: Split idt_data setup out of set_intr_gate() The code to setup idt_data is needed for early exception handling, but set_intr_gate() can't be used that early because it has pv-ops in its code path which don't work that early. Split out the idt_data initialization part from set_intr_gate() so that it can be used separately. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-29-joro@8bytes.org --- arch/x86/kernel/idt.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index 7ecf9babf0cb..53946c104fa0 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -205,18 +205,24 @@ idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sy } } +static void init_idt_data(struct idt_data *data, unsigned int n, + const void *addr) +{ + BUG_ON(n > 0xFF); + + memset(data, 0, sizeof(*data)); + data->vector = n; + data->addr = addr; + data->segment = __KERNEL_CS; + data->bits.type = GATE_INTERRUPT; + data->bits.p = 1; +} + static __init void set_intr_gate(unsigned int n, const void *addr) { struct idt_data data; - BUG_ON(n > 0xFF); - - memset(&data, 0, sizeof(data)); - data.vector = n; - data.addr = addr; - data.segment = __KERNEL_CS; - data.bits.type = GATE_INTERRUPT; - data.bits.p = 1; + init_idt_data(&data, n, addr); idt_setup_from_table(idt_table, &data, 1, false); } From 866b556efa1295934ed0bc20c2f208c93a873fb0 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:30 +0200 Subject: [PATCH 042/265] x86/head/64: Install startup GDT Handling exceptions during boot requires a working GDT. The kernel GDT can't be used on the direct mapping, so load a startup GDT and setup segments. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-30-joro@8bytes.org --- arch/x86/include/asm/setup.h | 1 + arch/x86/kernel/head64.c | 33 +++++++++++++++++++++++++++++++++ arch/x86/kernel/head_64.S | 14 ++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index 84b645cc8bc9..5c2fd05bd52c 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h @@ -48,6 +48,7 @@ extern void reserve_standard_io_resources(void); extern void i386_reserve_resources(void); extern unsigned long __startup_64(unsigned long physaddr, struct boot_params *bp); extern unsigned long __startup_secondary_64(void); +extern void startup_64_setup_env(unsigned long physbase); extern int early_make_pgtable(unsigned long address); #ifdef CONFIG_X86_INTEL_MID diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index cbb71c1b574f..8c82be44be94 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -61,6 +61,24 @@ unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4; EXPORT_SYMBOL(vmemmap_base); #endif +/* + * GDT used on the boot CPU before switching to virtual addresses. + */ +static struct desc_struct startup_gdt[GDT_ENTRIES] = { + [GDT_ENTRY_KERNEL32_CS] = GDT_ENTRY_INIT(0xc09b, 0, 0xfffff), + [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(0xa09b, 0, 0xfffff), + [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(0xc093, 0, 0xfffff), +}; + +/* + * Address needs to be set at runtime because it references the startup_gdt + * while the kernel still uses a direct mapping. + */ +static struct desc_ptr startup_gdt_descr = { + .size = sizeof(startup_gdt), + .address = 0, +}; + #define __head __section(.head.text) static void __head *fixup_pointer(void *ptr, unsigned long physaddr) @@ -489,3 +507,18 @@ void __init x86_64_start_reservations(char *real_mode_data) start_kernel(); } + +/* + * Setup boot CPU state needed before kernel switches to virtual addresses. + */ +void __head startup_64_setup_env(unsigned long physbase) +{ + /* Load GDT */ + startup_gdt_descr.address = (unsigned long)fixup_pointer(startup_gdt, physbase); + native_load_gdt(&startup_gdt_descr); + + /* New GDT is live - reload data segment registers */ + asm volatile("movl %%eax, %%ds\n" + "movl %%eax, %%ss\n" + "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory"); +} diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 16da4ac01597..2b2e91627221 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -73,6 +73,20 @@ SYM_CODE_START_NOALIGN(startup_64) /* Set up the stack for verify_cpu(), similar to initial_stack below */ leaq (__end_init_task - SIZEOF_PTREGS)(%rip), %rsp + leaq _text(%rip), %rdi + pushq %rsi + call startup_64_setup_env + popq %rsi + + /* Now switch to __KERNEL_CS so IRET works reliably */ + pushq $__KERNEL_CS + leaq .Lon_kernel_cs(%rip), %rax + pushq %rax + lretq + +.Lon_kernel_cs: + UNWIND_HINT_EMPTY + /* Sanitize CPU configuration */ call verify_cpu From e04b88336360e101329add0c05e5cb1cebae64fd Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:31 +0200 Subject: [PATCH 043/265] x86/head/64: Load GDT after switch to virtual addresses Load the GDT right after switching to virtual addresses to make sure there is a defined GDT for exception handling. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-31-joro@8bytes.org --- arch/x86/kernel/head_64.S | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 2b2e91627221..03b03f266dc1 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -158,6 +158,14 @@ SYM_CODE_START(secondary_startup_64) 1: UNWIND_HINT_EMPTY + /* + * We must switch to a new descriptor in kernel space for the GDT + * because soon the kernel won't have access anymore to the userspace + * addresses where we're currently running on. We have to do that here + * because in 32bit we couldn't load a 64bit linear address. + */ + lgdt early_gdt_descr(%rip) + /* Check if nx is implemented */ movl $0x80000001, %eax cpuid @@ -185,14 +193,6 @@ SYM_CODE_START(secondary_startup_64) pushq $0 popfq - /* - * We must switch to a new descriptor in kernel space for the GDT - * because soon the kernel won't have access anymore to the userspace - * addresses where we're currently running on. We have to do that here - * because in 32bit we couldn't load a 64bit linear address. - */ - lgdt early_gdt_descr(%rip) - /* set up data segments */ xorl %eax,%eax movl %eax,%ds From 7b99819dfb60268cc1c75f83c949bc4a09221bea Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:32 +0200 Subject: [PATCH 044/265] x86/head/64: Load segment registers earlier Make sure segments are properly set up before setting up an IDT and doing anything that might cause a #VC exception. This is later needed for early exception handling. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-32-joro@8bytes.org --- arch/x86/kernel/head_64.S | 52 +++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 03b03f266dc1..f402087a02ac 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -166,6 +166,32 @@ SYM_CODE_START(secondary_startup_64) */ lgdt early_gdt_descr(%rip) + /* set up data segments */ + xorl %eax,%eax + movl %eax,%ds + movl %eax,%ss + movl %eax,%es + + /* + * We don't really need to load %fs or %gs, but load them anyway + * to kill any stale realmode selectors. This allows execution + * under VT hardware. + */ + movl %eax,%fs + movl %eax,%gs + + /* Set up %gs. + * + * The base of %gs always points to fixed_percpu_data. If the + * stack protector canary is enabled, it is located at %gs:40. + * Note that, on SMP, the boot cpu uses init data section until + * the per cpu areas are set up. + */ + movl $MSR_GS_BASE,%ecx + movl initial_gs(%rip),%eax + movl initial_gs+4(%rip),%edx + wrmsr + /* Check if nx is implemented */ movl $0x80000001, %eax cpuid @@ -193,32 +219,6 @@ SYM_CODE_START(secondary_startup_64) pushq $0 popfq - /* set up data segments */ - xorl %eax,%eax - movl %eax,%ds - movl %eax,%ss - movl %eax,%es - - /* - * We don't really need to load %fs or %gs, but load them anyway - * to kill any stale realmode selectors. This allows execution - * under VT hardware. - */ - movl %eax,%fs - movl %eax,%gs - - /* Set up %gs. - * - * The base of %gs always points to fixed_percpu_data. If the - * stack protector canary is enabled, it is located at %gs:40. - * Note that, on SMP, the boot cpu uses init data section until - * the per cpu areas are set up. - */ - movl $MSR_GS_BASE,%ecx - movl initial_gs(%rip),%eax - movl initial_gs+4(%rip),%edx - wrmsr - /* rsi is pointer to real mode structure with interesting info. pass it to C */ movq %rsi, %rdi From 3add38cb96a1ae7d152db69ab4329e809c2af2d4 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:33 +0200 Subject: [PATCH 045/265] x86/head/64: Switch to initial stack earlier Make sure there is a stack once the kernel runs from virtual addresses. At this stage any secondary CPU which boots will have lost its stack because the kernel switched to a new page-table which does not map the real-mode stack anymore. This is needed for handling early #VC exceptions caused by instructions like CPUID. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-33-joro@8bytes.org --- arch/x86/kernel/head_64.S | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index f402087a02ac..83050c9e54d9 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -192,6 +192,12 @@ SYM_CODE_START(secondary_startup_64) movl initial_gs+4(%rip),%edx wrmsr + /* + * Setup a boot time stack - Any secondary CPU will have lost its stack + * by now because the cr3-switch above unmaps the real-mode stack + */ + movq initial_stack(%rip), %rsp + /* Check if nx is implemented */ movl $0x80000001, %eax cpuid @@ -212,9 +218,6 @@ SYM_CODE_START(secondary_startup_64) /* Make changes effective */ movq %rax, %cr0 - /* Setup a boot time stack */ - movq initial_stack(%rip), %rsp - /* zero EFLAGS after setting rsp */ pushq $0 popfq From f5963ba7a45fc6ff298a34976064354be437e1d8 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:34 +0200 Subject: [PATCH 046/265] x86/head/64: Install a CPU bringup IDT Add a separate bringup IDT for the CPU bringup code that will be used until the kernel switches to the idt_table. There are two reasons for a separate IDT: 1) When the idt_table is set up and the secondary CPUs are booted, it contains entries (e.g. IST entries) which require certain CPU state to be set up. This includes a working TSS (for IST), MSR_GS_BASE (for stack protector) or CR4.FSGSBASE (for paranoid_entry) path. By using a dedicated IDT for early boot this state need not to be set up early. 2) The idt_table is static to idt.c, so any function using/modifying must be in idt.c too. That means that all compiler driven instrumentation like tracing or KASAN is also active in this code. But during early CPU bringup the environment is not set up for this instrumentation to work correctly. To avoid all of these hassles and make early exception handling robust, use a dedicated bringup IDT. The IDT is loaded two times, first on the boot CPU while the kernel is still running on direct mapped addresses, and again later after the switch to kernel addresses has happened. The second IDT load happens on the boot and secondary CPUs. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-34-joro@8bytes.org --- arch/x86/include/asm/setup.h | 1 + arch/x86/kernel/head64.c | 39 ++++++++++++++++++++++++++++++++++++ arch/x86/kernel/head_64.S | 5 +++++ 3 files changed, 45 insertions(+) diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index 5c2fd05bd52c..4b3ca5ade2fd 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h @@ -50,6 +50,7 @@ extern unsigned long __startup_64(unsigned long physaddr, struct boot_params *bp extern unsigned long __startup_secondary_64(void); extern void startup_64_setup_env(unsigned long physbase); extern int early_make_pgtable(unsigned long address); +extern void early_setup_idt(void); #ifdef CONFIG_X86_INTEL_MID extern void x86_intel_mid_early_setup(void); diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 8c82be44be94..7bfd5c27c773 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include /* * Manage page tables very early on. @@ -508,6 +510,41 @@ void __init x86_64_start_reservations(char *real_mode_data) start_kernel(); } +/* + * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is + * used until the idt_table takes over. On the boot CPU this happens in + * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases + * this happens in the functions called from head_64.S. + * + * The idt_table can't be used that early because all the code modifying it is + * in idt.c and can be instrumented by tracing or KASAN, which both don't work + * during early CPU bringup. Also the idt_table has the runtime vectors + * configured which require certain CPU state to be setup already (like TSS), + * which also hasn't happened yet in early CPU bringup. + */ +static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data; + +static struct desc_ptr bringup_idt_descr = { + .size = (NUM_EXCEPTION_VECTORS * sizeof(gate_desc)) - 1, + .address = 0, /* Set at runtime */ +}; + +/* This runs while still in the direct mapping */ +static void startup_64_load_idt(unsigned long physbase) +{ + struct desc_ptr *desc = fixup_pointer(&bringup_idt_descr, physbase); + + desc->address = (unsigned long)fixup_pointer(bringup_idt_table, physbase); + native_load_idt(desc); +} + +/* This is used when running on kernel addresses */ +void early_setup_idt(void) +{ + bringup_idt_descr.address = (unsigned long)bringup_idt_table; + native_load_idt(&bringup_idt_descr); +} + /* * Setup boot CPU state needed before kernel switches to virtual addresses. */ @@ -521,4 +558,6 @@ void __head startup_64_setup_env(unsigned long physbase) asm volatile("movl %%eax, %%ds\n" "movl %%eax, %%ss\n" "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory"); + + startup_64_load_idt(physbase); } diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 83050c9e54d9..1de09b58e578 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -198,6 +198,11 @@ SYM_CODE_START(secondary_startup_64) */ movq initial_stack(%rip), %rsp + /* Setup and Load IDT */ + pushq %rsi + call early_setup_idt + popq %rsi + /* Check if nx is implemented */ movl $0x80000001, %eax cpuid From 097ee5b778b8970e1c2ed3ca1631b297d90acd61 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:35 +0200 Subject: [PATCH 047/265] x86/idt: Make IDT init functions static inlines Move these two functions from kernel/idt.c to include/asm/desc.h: * init_idt_data() * idt_init_desc() These functions are needed to setup IDT entries very early and need to be called from head64.c. To be usable this early, these functions need to be compiled without instrumentation and the stack-protector feature. These features need to be kept enabled for kernel/idt.c, so head64.c must use its own versions. [ bp: Take Kees' suggested patch title and add his Rev-by. ] Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-35-joro@8bytes.org --- arch/x86/include/asm/desc.h | 27 +++++++++++++++++++++++++ arch/x86/include/asm/desc_defs.h | 7 +++++++ arch/x86/kernel/idt.c | 34 -------------------------------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h index 1ced11d31932..476082a83d1c 100644 --- a/arch/x86/include/asm/desc.h +++ b/arch/x86/include/asm/desc.h @@ -383,6 +383,33 @@ static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit) void alloc_intr_gate(unsigned int n, const void *addr); +static inline void init_idt_data(struct idt_data *data, unsigned int n, + const void *addr) +{ + BUG_ON(n > 0xFF); + + memset(data, 0, sizeof(*data)); + data->vector = n; + data->addr = addr; + data->segment = __KERNEL_CS; + data->bits.type = GATE_INTERRUPT; + data->bits.p = 1; +} + +static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d) +{ + unsigned long addr = (unsigned long) d->addr; + + gate->offset_low = (u16) addr; + gate->segment = (u16) d->segment; + gate->bits = d->bits; + gate->offset_middle = (u16) (addr >> 16); +#ifdef CONFIG_X86_64 + gate->offset_high = (u32) (addr >> 32); + gate->reserved = 0; +#endif +} + extern unsigned long system_vectors[]; extern void load_current_idt(void); diff --git a/arch/x86/include/asm/desc_defs.h b/arch/x86/include/asm/desc_defs.h index 5621fb3f2d1a..f7e7099af595 100644 --- a/arch/x86/include/asm/desc_defs.h +++ b/arch/x86/include/asm/desc_defs.h @@ -74,6 +74,13 @@ struct idt_bits { p : 1; } __attribute__((packed)); +struct idt_data { + unsigned int vector; + unsigned int segment; + struct idt_bits bits; + const void *addr; +}; + struct gate_struct { u16 offset_low; u16 segment; diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index 53946c104fa0..4bb4e3d6099e 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -11,13 +11,6 @@ #include #include -struct idt_data { - unsigned int vector; - unsigned int segment; - struct idt_bits bits; - const void *addr; -}; - #define DPL0 0x0 #define DPL3 0x3 @@ -178,20 +171,6 @@ bool idt_is_f00f_address(unsigned long address) } #endif -static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d) -{ - unsigned long addr = (unsigned long) d->addr; - - gate->offset_low = (u16) addr; - gate->segment = (u16) d->segment; - gate->bits = d->bits; - gate->offset_middle = (u16) (addr >> 16); -#ifdef CONFIG_X86_64 - gate->offset_high = (u32) (addr >> 32); - gate->reserved = 0; -#endif -} - static __init void idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sys) { @@ -205,19 +184,6 @@ idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sy } } -static void init_idt_data(struct idt_data *data, unsigned int n, - const void *addr) -{ - BUG_ON(n > 0xFF); - - memset(data, 0, sizeof(*data)); - data->vector = n; - data->addr = addr; - data->segment = __KERNEL_CS; - data->bits.type = GATE_INTERRUPT; - data->bits.p = 1; -} - static __init void set_intr_gate(unsigned int n, const void *addr) { struct idt_data data; From 4b47cdbda6f1ad73b08dc7d497bac12b8f26ae0d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:36 +0200 Subject: [PATCH 048/265] x86/head/64: Move early exception dispatch to C code Move the assembly coded dispatch between page-faults and all other exceptions to C code to make it easier to maintain and extend. Also change the return-type of early_make_pgtable() to bool and make it static. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-36-joro@8bytes.org --- arch/x86/include/asm/pgtable.h | 2 +- arch/x86/include/asm/setup.h | 4 +++- arch/x86/kernel/head64.c | 19 +++++++++++++++---- arch/x86/kernel/head_64.S | 11 +---------- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index b836138ce852..7b8f2127de37 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -28,7 +28,7 @@ #include extern pgd_t early_top_pgt[PTRS_PER_PGD]; -int __init __early_make_pgtable(unsigned long address, pmdval_t pmd); +bool __init __early_make_pgtable(unsigned long address, pmdval_t pmd); void ptdump_walk_pgd_level(struct seq_file *m, struct mm_struct *mm); void ptdump_walk_pgd_level_debugfs(struct seq_file *m, struct mm_struct *mm, diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index 4b3ca5ade2fd..7d7a064af6ff 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h @@ -39,6 +39,8 @@ void vsmp_init(void); static inline void vsmp_init(void) { } #endif +struct pt_regs; + void setup_bios_corruption_check(void); void early_platform_quirks(void); @@ -49,8 +51,8 @@ extern void i386_reserve_resources(void); extern unsigned long __startup_64(unsigned long physaddr, struct boot_params *bp); extern unsigned long __startup_secondary_64(void); extern void startup_64_setup_env(unsigned long physbase); -extern int early_make_pgtable(unsigned long address); extern void early_setup_idt(void); +extern void __init do_early_exception(struct pt_regs *regs, int trapnr); #ifdef CONFIG_X86_INTEL_MID extern void x86_intel_mid_early_setup(void); diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 7bfd5c27c773..4282dac694c3 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -38,6 +38,8 @@ #include #include #include +#include +#include /* * Manage page tables very early on. @@ -317,7 +319,7 @@ static void __init reset_early_page_tables(void) } /* Create a new PMD entry */ -int __init __early_make_pgtable(unsigned long address, pmdval_t pmd) +bool __init __early_make_pgtable(unsigned long address, pmdval_t pmd) { unsigned long physaddr = address - __PAGE_OFFSET; pgdval_t pgd, *pgd_p; @@ -327,7 +329,7 @@ int __init __early_make_pgtable(unsigned long address, pmdval_t pmd) /* Invalid address or early pgt is done ? */ if (physaddr >= MAXMEM || read_cr3_pa() != __pa_nodebug(early_top_pgt)) - return -1; + return false; again: pgd_p = &early_top_pgt[pgd_index(address)].pgd; @@ -384,10 +386,10 @@ again: } pmd_p[pmd_index(address)] = pmd; - return 0; + return true; } -int __init early_make_pgtable(unsigned long address) +static bool __init early_make_pgtable(unsigned long address) { unsigned long physaddr = address - __PAGE_OFFSET; pmdval_t pmd; @@ -397,6 +399,15 @@ int __init early_make_pgtable(unsigned long address) return __early_make_pgtable(address, pmd); } +void __init do_early_exception(struct pt_regs *regs, int trapnr) +{ + if (trapnr == X86_TRAP_PF && + early_make_pgtable(native_read_cr2())) + return; + + early_fixup_exception(regs, trapnr); +} + /* Don't add a printk in there. printk relies on the PDA which is not initialized yet. */ static void __init clear_bss(void) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 1de09b58e578..3b40ec44a67d 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -341,18 +341,9 @@ SYM_CODE_START_LOCAL(early_idt_handler_common) pushq %r15 /* pt_regs->r15 */ UNWIND_HINT_REGS - cmpq $14,%rsi /* Page fault? */ - jnz 10f - GET_CR2_INTO(%rdi) /* can clobber %rax if pv */ - call early_make_pgtable - andl %eax,%eax - jz 20f /* All good */ - -10: movq %rsp,%rdi /* RDI = pt_regs; RSI is already trapnr */ - call early_fixup_exception + call do_early_exception -20: decl early_recursion_flag(%rip) jmp restore_regs_and_return_to_kernel SYM_CODE_END(early_idt_handler_common) From b57de6cd16395be1ebdaa9b489ffbf462bb585c4 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:37 +0200 Subject: [PATCH 049/265] x86/sev-es: Add SEV-ES Feature Detection Add a sev_es_active() function for checking whether SEV-ES is enabled. Also cache the value of MSR_AMD64_SEV at boot to speed up the feature checking in the running code. [ bp: Remove "!!" in sev_active() too. ] Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-37-joro@8bytes.org --- arch/x86/include/asm/mem_encrypt.h | 3 +++ arch/x86/include/asm/msr-index.h | 2 ++ arch/x86/mm/mem_encrypt.c | 9 ++++++++- arch/x86/mm/mem_encrypt_identity.c | 3 +++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h index 5049f6c22683..4e72b73a9cb5 100644 --- a/arch/x86/include/asm/mem_encrypt.h +++ b/arch/x86/include/asm/mem_encrypt.h @@ -19,6 +19,7 @@ #ifdef CONFIG_AMD_MEM_ENCRYPT extern u64 sme_me_mask; +extern u64 sev_status; extern bool sev_enabled; void sme_encrypt_execute(unsigned long encrypted_kernel_vaddr, @@ -50,6 +51,7 @@ void __init mem_encrypt_init(void); bool sme_active(void); bool sev_active(void); +bool sev_es_active(void); #define __bss_decrypted __attribute__((__section__(".bss..decrypted"))) @@ -72,6 +74,7 @@ static inline void __init sme_enable(struct boot_params *bp) { } static inline bool sme_active(void) { return false; } static inline bool sev_active(void) { return false; } +static inline bool sev_es_active(void) { return false; } static inline int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size) { return 0; } diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index da34fdba7c5a..249a4147c4b2 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -469,7 +469,9 @@ #define MSR_AMD64_SEV_ES_GHCB 0xc0010130 #define MSR_AMD64_SEV 0xc0010131 #define MSR_AMD64_SEV_ENABLED_BIT 0 +#define MSR_AMD64_SEV_ES_ENABLED_BIT 1 #define MSR_AMD64_SEV_ENABLED BIT_ULL(MSR_AMD64_SEV_ENABLED_BIT) +#define MSR_AMD64_SEV_ES_ENABLED BIT_ULL(MSR_AMD64_SEV_ES_ENABLED_BIT) #define MSR_AMD64_VIRT_SPEC_CTRL 0xc001011f diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index 9f1177edc2e7..a38f55676bb7 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -38,6 +38,7 @@ * section is later cleared. */ u64 sme_me_mask __section(.data) = 0; +u64 sev_status __section(.data) = 0; EXPORT_SYMBOL(sme_me_mask); DEFINE_STATIC_KEY_FALSE(sev_enable_key); EXPORT_SYMBOL_GPL(sev_enable_key); @@ -347,7 +348,13 @@ bool sme_active(void) bool sev_active(void) { - return sme_me_mask && sev_enabled; + return sev_status & MSR_AMD64_SEV_ENABLED; +} + +/* Needs to be called from non-instrumentable code */ +bool noinstr sev_es_active(void) +{ + return sev_status & MSR_AMD64_SEV_ES_ENABLED; } /* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */ diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c index e2b0e2ac07bb..68d75379e06a 100644 --- a/arch/x86/mm/mem_encrypt_identity.c +++ b/arch/x86/mm/mem_encrypt_identity.c @@ -540,6 +540,9 @@ void __init sme_enable(struct boot_params *bp) if (!(msr & MSR_AMD64_SEV_ENABLED)) return; + /* Save SEV_STATUS to avoid reading MSR again */ + sev_status = msr; + /* SEV state cannot be controlled by a command line option */ sme_me_mask = me_mask; sev_enabled = true; From c685eb0c12b4d4816d22ee734e91f4005b152fcd Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:38 +0200 Subject: [PATCH 050/265] x86/sev-es: Print SEV-ES info into the kernel log Refactor the message printed to the kernel log which indicates whether SEV or SME, etc is active. This will scale better in the future when more memory encryption features might be added. Also add SEV-ES to the list of features. [ bp: Massage. ] Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-38-joro@8bytes.org --- arch/x86/mm/mem_encrypt.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index a38f55676bb7..ebb7edc8bc0a 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -407,6 +407,31 @@ void __init mem_encrypt_free_decrypted_mem(void) free_init_pages("unused decrypted", vaddr, vaddr_end); } +static void print_mem_encrypt_feature_info(void) +{ + pr_info("AMD Memory Encryption Features active:"); + + /* Secure Memory Encryption */ + if (sme_active()) { + /* + * SME is mutually exclusive with any of the SEV + * features below. + */ + pr_cont(" SME\n"); + return; + } + + /* Secure Encrypted Virtualization */ + if (sev_active()) + pr_cont(" SEV"); + + /* Encrypted Register State */ + if (sev_es_active()) + pr_cont(" SEV-ES"); + + pr_cont("\n"); +} + /* Architecture __weak replacement functions */ void __init mem_encrypt_init(void) { @@ -422,8 +447,6 @@ void __init mem_encrypt_init(void) if (sev_active()) static_branch_enable(&sev_enable_key); - pr_info("AMD %s active\n", - sev_active() ? "Secure Encrypted Virtualization (SEV)" - : "Secure Memory Encryption (SME)"); + print_mem_encrypt_feature_info(); } From f980f9c31a923e9040dee0bc679a5f5b09e61f40 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:39 +0200 Subject: [PATCH 051/265] x86/sev-es: Compile early handler code into kernel image Setup sev-es.c and include the code from the pre-decompression stage to also build it into the image of the running kernel. Temporarily add __maybe_unused annotations to avoid build warnings until the functions get used. [ bp: Use the non-tracing rd/wrmsr variants because: vmlinux.o: warning: objtool: __sev_es_nmi_complete()+0x11f: \ call to do_trace_write_msr() leaves .noinstr.text section as __sev_es_nmi_complete() is noinstr due to being called from the NMI handler exc_nmi(). ] Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-39-joro@8bytes.org --- arch/x86/kernel/Makefile | 1 + arch/x86/kernel/sev-es-shared.c | 21 ++-- arch/x86/kernel/sev-es.c | 163 ++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 10 deletions(-) create mode 100644 arch/x86/kernel/sev-es.c diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index e77261db2391..23bc0f830f18 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -145,6 +145,7 @@ obj-$(CONFIG_UNWINDER_ORC) += unwind_orc.o obj-$(CONFIG_UNWINDER_FRAME_POINTER) += unwind_frame.o obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o +obj-$(CONFIG_AMD_MEM_ENCRYPT) += sev-es.o ### # 64 bit specific files ifeq ($(CONFIG_X86_64),y) diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index a6b41910b8ab..18619279a46f 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -9,7 +9,7 @@ * and is included directly into both code-bases. */ -static void sev_es_terminate(unsigned int reason) +static void __maybe_unused sev_es_terminate(unsigned int reason) { u64 val = GHCB_SEV_TERMINATE; @@ -27,7 +27,7 @@ static void sev_es_terminate(unsigned int reason) asm volatile("hlt\n" : : : "memory"); } -static bool sev_es_negotiate_protocol(void) +static bool __maybe_unused sev_es_negotiate_protocol(void) { u64 val; @@ -46,7 +46,7 @@ static bool sev_es_negotiate_protocol(void) return true; } -static void vc_ghcb_invalidate(struct ghcb *ghcb) +static void __maybe_unused vc_ghcb_invalidate(struct ghcb *ghcb) { memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); } @@ -58,9 +58,9 @@ static bool vc_decoding_needed(unsigned long exit_code) exit_code <= SVM_EXIT_LAST_EXCP); } -static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, - struct pt_regs *regs, - unsigned long exit_code) +static enum es_result __maybe_unused vc_init_em_ctxt(struct es_em_ctxt *ctxt, + struct pt_regs *regs, + unsigned long exit_code) { enum es_result ret = ES_OK; @@ -73,7 +73,7 @@ static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, return ret; } -static void vc_finish_insn(struct es_em_ctxt *ctxt) +static void __maybe_unused vc_finish_insn(struct es_em_ctxt *ctxt) { ctxt->regs->ip += ctxt->insn.length; } @@ -325,7 +325,8 @@ static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) return ES_OK; } -static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +static enum es_result __maybe_unused +vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; u64 exit_info_1, exit_info_2; @@ -433,8 +434,8 @@ static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) return ret; } -static enum es_result vc_handle_cpuid(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) +static enum es_result __maybe_unused vc_handle_cpuid(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; u32 cr4 = native_read_cr4(); diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c new file mode 100644 index 000000000000..4b1d217495f8 --- /dev/null +++ b/arch/x86/kernel/sev-es.c @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Memory Encryption Support + * + * Copyright (C) 2019 SUSE + * + * Author: Joerg Roedel + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +static inline u64 sev_es_rd_ghcb_msr(void) +{ + return __rdmsr(MSR_AMD64_SEV_ES_GHCB); +} + +static inline void sev_es_wr_ghcb_msr(u64 val) +{ + u32 low, high; + + low = (u32)(val); + high = (u32)(val >> 32); + + native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high); +} + +static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt, + unsigned char *buffer) +{ + return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); +} + +static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) +{ + char buffer[MAX_INSN_SIZE]; + enum es_result ret; + int res; + + res = vc_fetch_insn_kernel(ctxt, buffer); + if (unlikely(res == -EFAULT)) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = 0; + ctxt->fi.cr2 = ctxt->regs->ip; + return ES_EXCEPTION; + } + + insn_init(&ctxt->insn, buffer, MAX_INSN_SIZE - res, 1); + insn_get_length(&ctxt->insn); + + ret = ctxt->insn.immediate.got ? ES_OK : ES_DECODE_FAILED; + + return ret; +} + +static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, + char *dst, char *buf, size_t size) +{ + unsigned long error_code = X86_PF_PROT | X86_PF_WRITE; + char __user *target = (char __user *)dst; + u64 d8; + u32 d4; + u16 d2; + u8 d1; + + switch (size) { + case 1: + memcpy(&d1, buf, 1); + if (put_user(d1, target)) + goto fault; + break; + case 2: + memcpy(&d2, buf, 2); + if (put_user(d2, target)) + goto fault; + break; + case 4: + memcpy(&d4, buf, 4); + if (put_user(d4, target)) + goto fault; + break; + case 8: + memcpy(&d8, buf, 8); + if (put_user(d8, target)) + goto fault; + break; + default: + WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); + return ES_UNSUPPORTED; + } + + return ES_OK; + +fault: + if (user_mode(ctxt->regs)) + error_code |= X86_PF_USER; + + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = error_code; + ctxt->fi.cr2 = (unsigned long)dst; + + return ES_EXCEPTION; +} + +static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, + char *src, char *buf, size_t size) +{ + unsigned long error_code = X86_PF_PROT; + char __user *s = (char __user *)src; + u64 d8; + u32 d4; + u16 d2; + u8 d1; + + switch (size) { + case 1: + if (get_user(d1, s)) + goto fault; + memcpy(buf, &d1, 1); + break; + case 2: + if (get_user(d2, s)) + goto fault; + memcpy(buf, &d2, 2); + break; + case 4: + if (get_user(d4, s)) + goto fault; + memcpy(buf, &d4, 4); + break; + case 8: + if (get_user(d8, s)) + goto fault; + memcpy(buf, &d8, 8); + break; + default: + WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); + return ES_UNSUPPORTED; + } + + return ES_OK; + +fault: + if (user_mode(ctxt->regs)) + error_code |= X86_PF_USER; + + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = error_code; + ctxt->fi.cr2 = (unsigned long)src; + + return ES_EXCEPTION; +} + +/* Include code shared with pre-decompression boot stage */ +#include "sev-es-shared.c" From 74d8d9d531b4cc945a9f75aa2fc21d99ca5a9fe3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 8 Sep 2020 14:35:17 +0200 Subject: [PATCH 052/265] x86/sev-es: Setup an early #VC handler Setup an early handler for #VC exceptions. There is no GHCB mapped yet, so just re-use the vc_no_ghcb_handler(). It can only handle CPUID exit-codes, but that should be enough to get the kernel through verify_cpu() and __startup_64() until it runs on virtual addresses. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov [ boot failure Error: kernel_ident_mapping_init() failed. ] Reported-by: kernel test robot Link: https://lkml.kernel.org/r/20200908123517.GA3764@8bytes.org --- arch/x86/include/asm/sev-es.h | 3 +++ arch/x86/kernel/head64.c | 25 ++++++++++++++++++++++++- arch/x86/kernel/head_64.S | 30 ++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index 6dc52440c4b4..7175d432ebfe 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -73,4 +73,7 @@ static inline u64 lower_bits(u64 val, unsigned int bits) return (val & mask); } +/* Early IDT entry points for #VC handler */ +extern void vc_no_ghcb(void); + #endif diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 4282dac694c3..fc55cc9ccb0f 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -40,6 +40,7 @@ #include #include #include +#include /* * Manage page tables very early on. @@ -540,12 +541,34 @@ static struct desc_ptr bringup_idt_descr = { .address = 0, /* Set at runtime */ }; +static void set_bringup_idt_handler(gate_desc *idt, int n, void *handler) +{ +#ifdef CONFIG_AMD_MEM_ENCRYPT + struct idt_data data; + gate_desc desc; + + init_idt_data(&data, n, handler); + idt_init_desc(&desc, &data); + native_write_idt_entry(idt, n, &desc); +#endif +} + /* This runs while still in the direct mapping */ static void startup_64_load_idt(unsigned long physbase) { struct desc_ptr *desc = fixup_pointer(&bringup_idt_descr, physbase); + gate_desc *idt = fixup_pointer(bringup_idt_table, physbase); - desc->address = (unsigned long)fixup_pointer(bringup_idt_table, physbase); + + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { + void *handler; + + /* VMM Communication Exception */ + handler = fixup_pointer(vc_no_ghcb, physbase); + set_bringup_idt_handler(idt, X86_TRAP_VC, handler); + } + + desc->address = (unsigned long)idt; native_load_idt(desc); } diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 3b40ec44a67d..6e68bca64ae4 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -348,6 +348,36 @@ SYM_CODE_START_LOCAL(early_idt_handler_common) jmp restore_regs_and_return_to_kernel SYM_CODE_END(early_idt_handler_common) +#ifdef CONFIG_AMD_MEM_ENCRYPT +/* + * VC Exception handler used during very early boot. The + * early_idt_handler_array can't be used because it returns via the + * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early. + * + * This handler will end up in the .init.text section and not be + * available to boot secondary CPUs. + */ +SYM_CODE_START_NOALIGN(vc_no_ghcb) + UNWIND_HINT_IRET_REGS offset=8 + + /* Build pt_regs */ + PUSH_AND_CLEAR_REGS + + /* Call C handler */ + movq %rsp, %rdi + movq ORIG_RAX(%rsp), %rsi + call do_vc_no_ghcb + + /* Unwind pt_regs */ + POP_REGS + + /* Remove Error Code */ + addq $8, %rsp + + /* Pure iret required here - don't use INTERRUPT_RETURN */ + iretq +SYM_CODE_END(vc_no_ghcb) +#endif #define SYM_DATA_START_PAGE_ALIGNED(name) \ SYM_START(name, SYM_L_GLOBAL, .balign PAGE_SIZE) From 1aa9aa8ee517e0443b06e816a4fd2d15f2113615 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 8 Sep 2020 14:38:16 +0200 Subject: [PATCH 053/265] x86/sev-es: Setup GHCB-based boot #VC handler Add the infrastructure to handle #VC exceptions when the kernel runs on virtual addresses and has mapped a GHCB. This handler will be used until the runtime #VC handler takes over. Since the handler runs very early, disable instrumentation for sev-es.c. [ bp: Make vc_ghcb_invalidate() __always_inline so that it can be inlined in noinstr functions like __sev_es_nmi_complete(). ] Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200908123816.GB3764@8bytes.org --- arch/x86/include/asm/realmode.h | 3 + arch/x86/include/asm/segment.h | 2 +- arch/x86/include/asm/sev-es.h | 2 + arch/x86/kernel/Makefile | 2 + arch/x86/kernel/head64.c | 8 +++ arch/x86/kernel/head_64.S | 36 ++++++++++ arch/x86/kernel/sev-es-shared.c | 14 ++-- arch/x86/kernel/sev-es.c | 116 ++++++++++++++++++++++++++++++++ arch/x86/mm/extable.c | 1 + 9 files changed, 176 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/realmode.h b/arch/x86/include/asm/realmode.h index b35030eeec36..96118fb041b8 100644 --- a/arch/x86/include/asm/realmode.h +++ b/arch/x86/include/asm/realmode.h @@ -57,6 +57,9 @@ extern unsigned char real_mode_blob_end[]; extern unsigned long initial_code; extern unsigned long initial_gs; extern unsigned long initial_stack; +#ifdef CONFIG_AMD_MEM_ENCRYPT +extern unsigned long initial_vc_handler; +#endif extern unsigned char real_mode_blob[]; extern unsigned char real_mode_relocs[]; diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h index 9646c300f128..4e8dec387112 100644 --- a/arch/x86/include/asm/segment.h +++ b/arch/x86/include/asm/segment.h @@ -230,7 +230,7 @@ #define NUM_EXCEPTION_VECTORS 32 /* Bitmask of exception vectors which push an error code on the stack: */ -#define EXCEPTION_ERRCODE_MASK 0x00027d00 +#define EXCEPTION_ERRCODE_MASK 0x20027d00 #define GDT_SIZE (GDT_ENTRIES*8) #define GDT_ENTRY_TLS_ENTRIES 3 diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index 7175d432ebfe..9fbeedaa66ee 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -75,5 +75,7 @@ static inline u64 lower_bits(u64 val, unsigned int bits) /* Early IDT entry points for #VC handler */ extern void vc_no_ghcb(void); +extern void vc_boot_ghcb(void); +extern bool handle_vc_boot_ghcb(struct pt_regs *regs); #endif diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 23bc0f830f18..4a3327217bf0 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -20,6 +20,7 @@ CFLAGS_REMOVE_kvmclock.o = -pg CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_early_printk.o = -pg CFLAGS_REMOVE_head64.o = -pg +CFLAGS_REMOVE_sev-es.o = -pg endif KASAN_SANITIZE_head$(BITS).o := n @@ -27,6 +28,7 @@ KASAN_SANITIZE_dumpstack.o := n KASAN_SANITIZE_dumpstack_$(BITS).o := n KASAN_SANITIZE_stacktrace.o := n KASAN_SANITIZE_paravirt.o := n +KASAN_SANITIZE_sev-es.o := n # With some compiler versions the generated code results in boot hangs, caused # by several compilation units. To be safe, disable all instrumentation. diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index fc55cc9ccb0f..4199f25c0063 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -406,6 +406,10 @@ void __init do_early_exception(struct pt_regs *regs, int trapnr) early_make_pgtable(native_read_cr2())) return; + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT) && + trapnr == X86_TRAP_VC && handle_vc_boot_ghcb(regs)) + return; + early_fixup_exception(regs, trapnr); } @@ -575,6 +579,10 @@ static void startup_64_load_idt(unsigned long physbase) /* This is used when running on kernel addresses */ void early_setup_idt(void) { + /* VMM Communication Exception */ + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) + set_bringup_idt_handler(bringup_idt_table, X86_TRAP_VC, vc_boot_ghcb); + bringup_idt_descr.address = (unsigned long)bringup_idt_table; native_load_idt(&bringup_idt_descr); } diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 6e68bca64ae4..1a71d0d4d575 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -279,6 +279,39 @@ SYM_CODE_START(start_cpu0) movq initial_stack(%rip), %rsp jmp .Ljump_to_C_code SYM_CODE_END(start_cpu0) +#endif + +#ifdef CONFIG_AMD_MEM_ENCRYPT +/* + * VC Exception handler used during early boot when running on kernel + * addresses, but before the switch to the idt_table can be made. + * The early_idt_handler_array can't be used here because it calls into a lot + * of __init code and this handler is also used during CPU offlining/onlining. + * Therefore this handler ends up in the .text section so that it stays around + * when .init.text is freed. + */ +SYM_CODE_START_NOALIGN(vc_boot_ghcb) + UNWIND_HINT_IRET_REGS offset=8 + + /* Build pt_regs */ + PUSH_AND_CLEAR_REGS + + /* Call C handler */ + movq %rsp, %rdi + movq ORIG_RAX(%rsp), %rsi + movq initial_vc_handler(%rip), %rax + ANNOTATE_RETPOLINE_SAFE + call *%rax + + /* Unwind pt_regs */ + POP_REGS + + /* Remove Error Code */ + addq $8, %rsp + + /* Pure iret required here - don't use INTERRUPT_RETURN */ + iretq +SYM_CODE_END(vc_boot_ghcb) #endif /* Both SMP bootup and ACPI suspend change these variables */ @@ -286,6 +319,9 @@ SYM_CODE_END(start_cpu0) .balign 8 SYM_DATA(initial_code, .quad x86_64_start_kernel) SYM_DATA(initial_gs, .quad INIT_PER_CPU_VAR(fixed_percpu_data)) +#ifdef CONFIG_AMD_MEM_ENCRYPT +SYM_DATA(initial_vc_handler, .quad handle_vc_boot_ghcb) +#endif /* * The SIZEOF_PTREGS gap is a convention which helps the in-kernel unwinder diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 18619279a46f..53e3de0e8060 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -9,7 +9,7 @@ * and is included directly into both code-bases. */ -static void __maybe_unused sev_es_terminate(unsigned int reason) +static void sev_es_terminate(unsigned int reason) { u64 val = GHCB_SEV_TERMINATE; @@ -27,7 +27,7 @@ static void __maybe_unused sev_es_terminate(unsigned int reason) asm volatile("hlt\n" : : : "memory"); } -static bool __maybe_unused sev_es_negotiate_protocol(void) +static bool sev_es_negotiate_protocol(void) { u64 val; @@ -46,7 +46,7 @@ static bool __maybe_unused sev_es_negotiate_protocol(void) return true; } -static void __maybe_unused vc_ghcb_invalidate(struct ghcb *ghcb) +static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) { memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); } @@ -58,9 +58,9 @@ static bool vc_decoding_needed(unsigned long exit_code) exit_code <= SVM_EXIT_LAST_EXCP); } -static enum es_result __maybe_unused vc_init_em_ctxt(struct es_em_ctxt *ctxt, - struct pt_regs *regs, - unsigned long exit_code) +static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, + struct pt_regs *regs, + unsigned long exit_code) { enum es_result ret = ES_OK; @@ -73,7 +73,7 @@ static enum es_result __maybe_unused vc_init_em_ctxt(struct es_em_ctxt *ctxt, return ret; } -static void __maybe_unused vc_finish_insn(struct es_em_ctxt *ctxt) +static void vc_finish_insn(struct es_em_ctxt *ctxt) { ctxt->regs->ip += ctxt->insn.length; } diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 4b1d217495f8..213a42774aa9 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -7,7 +7,9 @@ * Author: Joerg Roedel */ +#include /* For show_regs() */ #include +#include #include #include @@ -18,6 +20,18 @@ #include #include +/* For early boot hypervisor communication in SEV-ES enabled guests */ +static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); + +/* + * Needs to be in the .data section because we need it NULL before bss is + * cleared + */ +static struct ghcb __initdata *boot_ghcb; + +/* Needed in vc_early_forward_exception */ +void do_early_exception(struct pt_regs *regs, int trapnr); + static inline u64 sev_es_rd_ghcb_msr(void) { return __rdmsr(MSR_AMD64_SEV_ES_GHCB); @@ -161,3 +175,105 @@ fault: /* Include code shared with pre-decompression boot stage */ #include "sev-es-shared.c" + +/* + * This function runs on the first #VC exception after the kernel + * switched to virtual addresses. + */ +static bool __init sev_es_setup_ghcb(void) +{ + /* First make sure the hypervisor talks a supported protocol. */ + if (!sev_es_negotiate_protocol()) + return false; + + /* + * Clear the boot_ghcb. The first exception comes in before the bss + * section is cleared. + */ + memset(&boot_ghcb_page, 0, PAGE_SIZE); + + /* Alright - Make the boot-ghcb public */ + boot_ghcb = &boot_ghcb_page; + + return true; +} + +static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) +{ + int trapnr = ctxt->fi.vector; + + if (trapnr == X86_TRAP_PF) + native_write_cr2(ctxt->fi.cr2); + + ctxt->regs->orig_ax = ctxt->fi.error_code; + do_early_exception(ctxt->regs, trapnr); +} + +static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, + struct ghcb *ghcb, + unsigned long exit_code) +{ + enum es_result result; + + switch (exit_code) { + default: + /* + * Unexpected #VC exception + */ + result = ES_UNSUPPORTED; + } + + return result; +} + +bool __init handle_vc_boot_ghcb(struct pt_regs *regs) +{ + unsigned long exit_code = regs->orig_ax; + struct es_em_ctxt ctxt; + enum es_result result; + + /* Do initial setup or terminate the guest */ + if (unlikely(boot_ghcb == NULL && !sev_es_setup_ghcb())) + sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); + + vc_ghcb_invalidate(boot_ghcb); + + result = vc_init_em_ctxt(&ctxt, regs, exit_code); + if (result == ES_OK) + result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code); + + /* Done - now check the result */ + switch (result) { + case ES_OK: + vc_finish_insn(&ctxt); + break; + case ES_UNSUPPORTED: + early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n", + exit_code, regs->ip); + goto fail; + case ES_VMM_ERROR: + early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", + exit_code, regs->ip); + goto fail; + case ES_DECODE_FAILED: + early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", + exit_code, regs->ip); + goto fail; + case ES_EXCEPTION: + vc_early_forward_exception(&ctxt); + break; + case ES_RETRY: + /* Nothing to do */ + break; + default: + BUG(); + } + + return true; + +fail: + show_regs(regs); + + while (true) + halt(); +} diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index 1d6cb07f4f86..3966749d07ac 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -5,6 +5,7 @@ #include #include +#include #include #include From 885689e47dfa1499b756a07237eb645234d93cf9 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:42 +0200 Subject: [PATCH 054/265] x86/sev-es: Setup per-CPU GHCBs for the runtime handler The runtime handler needs one GHCB per-CPU. Set them up and map them unencrypted. [ bp: Touchups and simplification. ] Signed-off-by: Tom Lendacky Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-42-joro@8bytes.org --- arch/x86/include/asm/mem_encrypt.h | 2 ++ arch/x86/kernel/sev-es.c | 56 +++++++++++++++++++++++++++++- arch/x86/kernel/traps.c | 3 ++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h index 4e72b73a9cb5..c9f5df0a1c10 100644 --- a/arch/x86/include/asm/mem_encrypt.h +++ b/arch/x86/include/asm/mem_encrypt.h @@ -49,6 +49,7 @@ void __init mem_encrypt_free_decrypted_mem(void); /* Architecture __weak replacement functions */ void __init mem_encrypt_init(void); +void __init sev_es_init_vc_handling(void); bool sme_active(void); bool sev_active(void); bool sev_es_active(void); @@ -72,6 +73,7 @@ static inline void __init sme_early_init(void) { } static inline void __init sme_encrypt_kernel(struct boot_params *bp) { } static inline void __init sme_enable(struct boot_params *bp) { } +static inline void sev_es_init_vc_handling(void) { } static inline bool sme_active(void) { return false; } static inline bool sev_active(void) { return false; } static inline bool sev_es_active(void) { return false; } diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 213a42774aa9..720b1b671812 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -8,8 +8,13 @@ */ #include /* For show_regs() */ -#include +#include +#include #include +#include +#include +#include +#include #include #include @@ -29,6 +34,13 @@ static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); */ static struct ghcb __initdata *boot_ghcb; +/* #VC handler runtime per-CPU data */ +struct sev_es_runtime_data { + struct ghcb ghcb_page; +}; + +static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); + /* Needed in vc_early_forward_exception */ void do_early_exception(struct pt_regs *regs, int trapnr); @@ -198,6 +210,48 @@ static bool __init sev_es_setup_ghcb(void) return true; } +static void __init alloc_runtime_data(int cpu) +{ + struct sev_es_runtime_data *data; + + data = memblock_alloc(sizeof(*data), PAGE_SIZE); + if (!data) + panic("Can't allocate SEV-ES runtime data"); + + per_cpu(runtime_data, cpu) = data; +} + +static void __init init_ghcb(int cpu) +{ + struct sev_es_runtime_data *data; + int err; + + data = per_cpu(runtime_data, cpu); + + err = early_set_memory_decrypted((unsigned long)&data->ghcb_page, + sizeof(data->ghcb_page)); + if (err) + panic("Can't map GHCBs unencrypted"); + + memset(&data->ghcb_page, 0, sizeof(data->ghcb_page)); +} + +void __init sev_es_init_vc_handling(void) +{ + int cpu; + + BUILD_BUG_ON(offsetof(struct sev_es_runtime_data, ghcb_page) % PAGE_SIZE); + + if (!sev_es_active()) + return; + + /* Initialize per-cpu GHCB pages */ + for_each_possible_cpu(cpu) { + alloc_runtime_data(cpu); + init_ghcb(cpu); + } +} + static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) { int trapnr = ctxt->fi.vector; diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 81a2fb711091..2dc32b0ca38e 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -1074,6 +1074,9 @@ void __init trap_init(void) /* Init cpu_entry_area before IST entries are set up */ setup_cpu_entry_areas(); + /* Init GHCB memory pages when running as an SEV-ES guest */ + sev_es_init_vc_handling(); + idt_setup_traps(); /* From 02772fb9b68e6a72a5e17f994048df832fe2b15e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:43 +0200 Subject: [PATCH 055/265] x86/sev-es: Allocate and map an IST stack for #VC handler Allocate and map an IST stack and an additional fall-back stack for the #VC handler. The memory for the stacks is allocated only when SEV-ES is active. The #VC handler needs to use an IST stack because a #VC exception can be raised from kernel space with unsafe stack, e.g. in the SYSCALL entry path. Since the #VC exception can be nested, the #VC handler switches back to the interrupted stack when entered from kernel space. If switching back is not possible, the fall-back stack is used. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-43-joro@8bytes.org --- arch/x86/include/asm/cpu_entry_area.h | 33 +++++++++++++++++---------- arch/x86/include/asm/page_64_types.h | 1 + arch/x86/kernel/cpu/common.c | 2 ++ arch/x86/kernel/dumpstack_64.c | 8 +++++-- arch/x86/kernel/sev-es.c | 33 +++++++++++++++++++++++++++ 5 files changed, 63 insertions(+), 14 deletions(-) diff --git a/arch/x86/include/asm/cpu_entry_area.h b/arch/x86/include/asm/cpu_entry_area.h index 8902fdb7de13..3d52b094850a 100644 --- a/arch/x86/include/asm/cpu_entry_area.h +++ b/arch/x86/include/asm/cpu_entry_area.h @@ -11,25 +11,29 @@ #ifdef CONFIG_X86_64 /* Macro to enforce the same ordering and stack sizes */ -#define ESTACKS_MEMBERS(guardsize) \ - char DF_stack_guard[guardsize]; \ - char DF_stack[EXCEPTION_STKSZ]; \ - char NMI_stack_guard[guardsize]; \ - char NMI_stack[EXCEPTION_STKSZ]; \ - char DB_stack_guard[guardsize]; \ - char DB_stack[EXCEPTION_STKSZ]; \ - char MCE_stack_guard[guardsize]; \ - char MCE_stack[EXCEPTION_STKSZ]; \ - char IST_top_guard[guardsize]; \ +#define ESTACKS_MEMBERS(guardsize, optional_stack_size) \ + char DF_stack_guard[guardsize]; \ + char DF_stack[EXCEPTION_STKSZ]; \ + char NMI_stack_guard[guardsize]; \ + char NMI_stack[EXCEPTION_STKSZ]; \ + char DB_stack_guard[guardsize]; \ + char DB_stack[EXCEPTION_STKSZ]; \ + char MCE_stack_guard[guardsize]; \ + char MCE_stack[EXCEPTION_STKSZ]; \ + char VC_stack_guard[guardsize]; \ + char VC_stack[optional_stack_size]; \ + char VC2_stack_guard[guardsize]; \ + char VC2_stack[optional_stack_size]; \ + char IST_top_guard[guardsize]; \ /* The exception stacks' physical storage. No guard pages required */ struct exception_stacks { - ESTACKS_MEMBERS(0) + ESTACKS_MEMBERS(0, 0) }; /* The effective cpu entry area mapping with guard pages. */ struct cea_exception_stacks { - ESTACKS_MEMBERS(PAGE_SIZE) + ESTACKS_MEMBERS(PAGE_SIZE, EXCEPTION_STKSZ) }; /* @@ -40,6 +44,8 @@ enum exception_stack_ordering { ESTACK_NMI, ESTACK_DB, ESTACK_MCE, + ESTACK_VC, + ESTACK_VC2, N_EXCEPTION_STACKS }; @@ -139,4 +145,7 @@ static inline struct entry_stack *cpu_entry_stack(int cpu) #define __this_cpu_ist_top_va(name) \ CEA_ESTACK_TOP(__this_cpu_read(cea_exception_stacks), name) +#define __this_cpu_ist_bottom_va(name) \ + CEA_ESTACK_BOT(__this_cpu_read(cea_exception_stacks), name) + #endif diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h index 288b065955b7..d0c6c10c18a0 100644 --- a/arch/x86/include/asm/page_64_types.h +++ b/arch/x86/include/asm/page_64_types.h @@ -28,6 +28,7 @@ #define IST_INDEX_NMI 1 #define IST_INDEX_DB 2 #define IST_INDEX_MCE 3 +#define IST_INDEX_VC 4 /* * Set __PAGE_OFFSET to the most negative possible address + diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index c5d6f17d9b9d..81fba4d4a242 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1829,6 +1829,8 @@ static inline void tss_setup_ist(struct tss_struct *tss) tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI); tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB); tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE); + /* Only mapped when SEV-ES is active */ + tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC); } #else /* CONFIG_X86_64 */ diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index 4a94d38cd141..c49cf594714b 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -24,11 +24,13 @@ static const char * const exception_stack_names[] = { [ ESTACK_NMI ] = "NMI", [ ESTACK_DB ] = "#DB", [ ESTACK_MCE ] = "#MC", + [ ESTACK_VC ] = "#VC", + [ ESTACK_VC2 ] = "#VC2", }; const char *stack_type_name(enum stack_type type) { - BUILD_BUG_ON(N_EXCEPTION_STACKS != 4); + BUILD_BUG_ON(N_EXCEPTION_STACKS != 6); if (type == STACK_TYPE_IRQ) return "IRQ"; @@ -79,6 +81,8 @@ struct estack_pages estack_pages[CEA_ESTACK_PAGES] ____cacheline_aligned = { EPAGERANGE(NMI), EPAGERANGE(DB), EPAGERANGE(MCE), + EPAGERANGE(VC), + EPAGERANGE(VC2), }; static bool in_exception_stack(unsigned long *stack, struct stack_info *info) @@ -88,7 +92,7 @@ static bool in_exception_stack(unsigned long *stack, struct stack_info *info) struct pt_regs *regs; unsigned int k; - BUILD_BUG_ON(N_EXCEPTION_STACKS != 4); + BUILD_BUG_ON(N_EXCEPTION_STACKS != 6); begin = (unsigned long)__this_cpu_read(cea_exception_stacks); /* diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 720b1b671812..fae8145e4acf 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -37,10 +38,41 @@ static struct ghcb __initdata *boot_ghcb; /* #VC handler runtime per-CPU data */ struct sev_es_runtime_data { struct ghcb ghcb_page; + + /* Physical storage for the per-CPU IST stack of the #VC handler */ + char ist_stack[EXCEPTION_STKSZ] __aligned(PAGE_SIZE); + + /* + * Physical storage for the per-CPU fall-back stack of the #VC handler. + * The fall-back stack is used when it is not safe to switch back to the + * interrupted stack in the #VC entry code. + */ + char fallback_stack[EXCEPTION_STKSZ] __aligned(PAGE_SIZE); }; static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); +static void __init setup_vc_stacks(int cpu) +{ + struct sev_es_runtime_data *data; + struct cpu_entry_area *cea; + unsigned long vaddr; + phys_addr_t pa; + + data = per_cpu(runtime_data, cpu); + cea = get_cpu_entry_area(cpu); + + /* Map #VC IST stack */ + vaddr = CEA_ESTACK_BOT(&cea->estacks, VC); + pa = __pa(data->ist_stack); + cea_set_pte((void *)vaddr, pa, PAGE_KERNEL); + + /* Map VC fall-back stack */ + vaddr = CEA_ESTACK_BOT(&cea->estacks, VC2); + pa = __pa(data->fallback_stack); + cea_set_pte((void *)vaddr, pa, PAGE_KERNEL); +} + /* Needed in vc_early_forward_exception */ void do_early_exception(struct pt_regs *regs, int trapnr); @@ -249,6 +281,7 @@ void __init sev_es_init_vc_handling(void) for_each_possible_cpu(cpu) { alloc_runtime_data(cpu); init_ghcb(cpu); + setup_vc_stacks(cpu); } } From 315562c9af3d583502b35c4b223a08d95ce69864 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:44 +0200 Subject: [PATCH 056/265] x86/sev-es: Adjust #VC IST Stack on entering NMI handler When an NMI hits in the #VC handler entry code before it has switched to another stack, any subsequent #VC exception in the NMI code-path will overwrite the interrupted #VC handler's stack. Make sure this doesn't happen by explicitly adjusting the #VC IST entry in the NMI handler for the time it can cause #VC exceptions. [ bp: Touchups, spelling fixes. ] Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-44-joro@8bytes.org --- arch/x86/include/asm/sev-es.h | 19 +++++++++++++ arch/x86/kernel/nmi.c | 9 ++++++ arch/x86/kernel/sev-es.c | 53 +++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index 9fbeedaa66ee..59176e8c6b81 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -78,4 +78,23 @@ extern void vc_no_ghcb(void); extern void vc_boot_ghcb(void); extern bool handle_vc_boot_ghcb(struct pt_regs *regs); +#ifdef CONFIG_AMD_MEM_ENCRYPT +extern struct static_key_false sev_es_enable_key; +extern void __sev_es_ist_enter(struct pt_regs *regs); +extern void __sev_es_ist_exit(void); +static __always_inline void sev_es_ist_enter(struct pt_regs *regs) +{ + if (static_branch_unlikely(&sev_es_enable_key)) + __sev_es_ist_enter(regs); +} +static __always_inline void sev_es_ist_exit(void) +{ + if (static_branch_unlikely(&sev_es_enable_key)) + __sev_es_ist_exit(); +} +#else +static inline void sev_es_ist_enter(struct pt_regs *regs) { } +static inline void sev_es_ist_exit(void) { } +#endif + #endif diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 4fc9954a9560..4c89c4d3cb82 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -33,6 +33,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -488,6 +489,12 @@ DEFINE_IDTENTRY_RAW(exc_nmi) this_cpu_write(nmi_cr2, read_cr2()); nmi_restart: + /* + * Needs to happen before DR7 is accessed, because the hypervisor can + * intercept DR7 reads/writes, turning those into #VC exceptions. + */ + sev_es_ist_enter(regs); + this_cpu_write(nmi_dr7, local_db_save()); irq_state = idtentry_enter_nmi(regs); @@ -501,6 +508,8 @@ nmi_restart: local_db_restore(this_cpu_read(nmi_dr7)); + sev_es_ist_exit(); + if (unlikely(this_cpu_read(nmi_cr2) != read_cr2())) write_cr2(this_cpu_read(nmi_cr2)); if (this_cpu_dec_return(nmi_state)) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index fae8145e4acf..39ebb2d04bec 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -51,6 +51,7 @@ struct sev_es_runtime_data { }; static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); +DEFINE_STATIC_KEY_FALSE(sev_es_enable_key); static void __init setup_vc_stacks(int cpu) { @@ -73,6 +74,55 @@ static void __init setup_vc_stacks(int cpu) cea_set_pte((void *)vaddr, pa, PAGE_KERNEL); } +static __always_inline bool on_vc_stack(unsigned long sp) +{ + return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC))); +} + +/* + * This function handles the case when an NMI is raised in the #VC exception + * handler entry code. In this case, the IST entry for #VC must be adjusted, so + * that any subsequent #VC exception will not overwrite the stack contents of the + * interrupted #VC handler. + * + * The IST entry is adjusted unconditionally so that it can be also be + * unconditionally adjusted back in sev_es_ist_exit(). Otherwise a nested + * sev_es_ist_exit() call may adjust back the IST entry too early. + */ +void noinstr __sev_es_ist_enter(struct pt_regs *regs) +{ + unsigned long old_ist, new_ist; + + /* Read old IST entry */ + old_ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]); + + /* Make room on the IST stack */ + if (on_vc_stack(regs->sp)) + new_ist = ALIGN_DOWN(regs->sp, 8) - sizeof(old_ist); + else + new_ist = old_ist - sizeof(old_ist); + + /* Store old IST entry */ + *(unsigned long *)new_ist = old_ist; + + /* Set new IST entry */ + this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], new_ist); +} + +void noinstr __sev_es_ist_exit(void) +{ + unsigned long ist; + + /* Read IST entry */ + ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]); + + if (WARN_ON(ist == __this_cpu_ist_top_va(VC))) + return; + + /* Read back old IST entry and write it to the TSS */ + this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist); +} + /* Needed in vc_early_forward_exception */ void do_early_exception(struct pt_regs *regs, int trapnr); @@ -277,6 +327,9 @@ void __init sev_es_init_vc_handling(void) if (!sev_es_active()) return; + /* Enable SEV-ES special handling */ + static_branch_enable(&sev_es_enable_key); + /* Initialize per-cpu GHCB pages */ for_each_possible_cpu(cpu) { alloc_runtime_data(cpu); From 6b27edd74a5e9669120f7bd0ae1f475d124c1042 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:45 +0200 Subject: [PATCH 057/265] x86/dumpstack/64: Add noinstr version of get_stack_info() The get_stack_info() functionality is needed in the entry code for the #VC exception handler. Provide a version of it in the .text.noinstr section which can be called safely from there. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-45-joro@8bytes.org --- arch/x86/include/asm/stacktrace.h | 2 ++ arch/x86/kernel/dumpstack.c | 7 +++-- arch/x86/kernel/dumpstack_64.c | 46 +++++++++++++++++-------------- arch/x86/mm/cpu_entry_area.c | 3 +- 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h index 5ae5a68e469d..49600643faba 100644 --- a/arch/x86/include/asm/stacktrace.h +++ b/arch/x86/include/asm/stacktrace.h @@ -35,6 +35,8 @@ bool in_entry_stack(unsigned long *stack, struct stack_info *info); int get_stack_info(unsigned long *stack, struct task_struct *task, struct stack_info *info, unsigned long *visit_mask); +bool get_stack_info_noinstr(unsigned long *stack, struct task_struct *task, + struct stack_info *info); const char *stack_type_name(enum stack_type type); diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index 48ce44576947..74147f7b3d82 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -29,8 +29,8 @@ static int die_counter; static struct pt_regs exec_summary_regs; -bool in_task_stack(unsigned long *stack, struct task_struct *task, - struct stack_info *info) +bool noinstr in_task_stack(unsigned long *stack, struct task_struct *task, + struct stack_info *info) { unsigned long *begin = task_stack_page(task); unsigned long *end = task_stack_page(task) + THREAD_SIZE; @@ -46,7 +46,8 @@ bool in_task_stack(unsigned long *stack, struct task_struct *task, return true; } -bool in_entry_stack(unsigned long *stack, struct stack_info *info) +/* Called from get_stack_info_noinstr - so must be noinstr too */ +bool noinstr in_entry_stack(unsigned long *stack, struct stack_info *info) { struct entry_stack *ss = cpu_entry_stack(smp_processor_id()); diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index c49cf594714b..1dd851397bd9 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -85,7 +85,7 @@ struct estack_pages estack_pages[CEA_ESTACK_PAGES] ____cacheline_aligned = { EPAGERANGE(VC2), }; -static bool in_exception_stack(unsigned long *stack, struct stack_info *info) +static __always_inline bool in_exception_stack(unsigned long *stack, struct stack_info *info) { unsigned long begin, end, stk = (unsigned long)stack; const struct estack_pages *ep; @@ -126,7 +126,7 @@ static bool in_exception_stack(unsigned long *stack, struct stack_info *info) return true; } -static bool in_irq_stack(unsigned long *stack, struct stack_info *info) +static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info *info) { unsigned long *end = (unsigned long *)this_cpu_read(hardirq_stack_ptr); unsigned long *begin = end - (IRQ_STACK_SIZE / sizeof(long)); @@ -151,32 +151,38 @@ static bool in_irq_stack(unsigned long *stack, struct stack_info *info) return true; } +bool noinstr get_stack_info_noinstr(unsigned long *stack, struct task_struct *task, + struct stack_info *info) +{ + if (in_task_stack(stack, task, info)) + return true; + + if (task != current) + return false; + + if (in_exception_stack(stack, info)) + return true; + + if (in_irq_stack(stack, info)) + return true; + + if (in_entry_stack(stack, info)) + return true; + + return false; +} + int get_stack_info(unsigned long *stack, struct task_struct *task, struct stack_info *info, unsigned long *visit_mask) { + task = task ? : current; + if (!stack) goto unknown; - task = task ? : current; - - if (in_task_stack(stack, task, info)) - goto recursion_check; - - if (task != current) + if (!get_stack_info_noinstr(stack, task, info)) goto unknown; - if (in_exception_stack(stack, info)) - goto recursion_check; - - if (in_irq_stack(stack, info)) - goto recursion_check; - - if (in_entry_stack(stack, info)) - goto recursion_check; - - goto unknown; - -recursion_check: /* * Make sure we don't iterate through any given stack more than once. * If it comes up a second time then there's something wrong going on: diff --git a/arch/x86/mm/cpu_entry_area.c b/arch/x86/mm/cpu_entry_area.c index 770b613790b3..f5e1e60c9095 100644 --- a/arch/x86/mm/cpu_entry_area.c +++ b/arch/x86/mm/cpu_entry_area.c @@ -21,7 +21,8 @@ DEFINE_PER_CPU(struct cea_exception_stacks*, cea_exception_stacks); DECLARE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack); #endif -struct cpu_entry_area *get_cpu_entry_area(int cpu) +/* Is called from entry code, so must be noinstr */ +noinstr struct cpu_entry_area *get_cpu_entry_area(int cpu) { unsigned long va = CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE; BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0); From a13644f3a53de4e95a7bce6459f834e832ea44c5 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:46 +0200 Subject: [PATCH 058/265] x86/entry/64: Add entry code for #VC handler The #VC handler needs special entry code because: 1. It runs on an IST stack 2. It needs to be able to handle nested #VC exceptions To make this work, the entry code is implemented to pretend it doesn't use an IST stack. When entered from user-mode or early SYSCALL entry path it switches to the task stack. If entered from kernel-mode it tries to switch back to the previous stack in the IRET frame. The stack found in the IRET frame is validated first, and if it is not safe to use it for the #VC handler, the code will switch to a fall-back stack (the #VC2 IST stack). From there, it can cause nested exceptions again. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-46-joro@8bytes.org --- arch/x86/entry/entry_64.S | 80 +++++++++++++++++++++++++++++++++ arch/x86/include/asm/idtentry.h | 44 ++++++++++++++++++ arch/x86/include/asm/proto.h | 1 + arch/x86/include/asm/traps.h | 1 + arch/x86/kernel/traps.c | 45 +++++++++++++++++++ 5 files changed, 171 insertions(+) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 70dea9337816..15aa1896927c 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -101,6 +101,8 @@ SYM_CODE_START(entry_SYSCALL_64) SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp +SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL) + /* Construct struct pt_regs on stack */ pushq $__USER_DS /* pt_regs->ss */ pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2) /* pt_regs->sp */ @@ -446,6 +448,84 @@ _ASM_NOKPROBE(\asmsym) SYM_CODE_END(\asmsym) .endm +#ifdef CONFIG_AMD_MEM_ENCRYPT +/** + * idtentry_vc - Macro to generate entry stub for #VC + * @vector: Vector number + * @asmsym: ASM symbol for the entry point + * @cfunc: C function to be called + * + * The macro emits code to set up the kernel context for #VC. The #VC handler + * runs on an IST stack and needs to be able to cause nested #VC exceptions. + * + * To make this work the #VC entry code tries its best to pretend it doesn't use + * an IST stack by switching to the task stack if coming from user-space (which + * includes early SYSCALL entry path) or back to the stack in the IRET frame if + * entered from kernel-mode. + * + * If entered from kernel-mode the return stack is validated first, and if it is + * not safe to use (e.g. because it points to the entry stack) the #VC handler + * will switch to a fall-back stack (VC2) and call a special handler function. + * + * The macro is only used for one vector, but it is planned to be extended in + * the future for the #HV exception. + */ +.macro idtentry_vc vector asmsym cfunc +SYM_CODE_START(\asmsym) + UNWIND_HINT_IRET_REGS + ASM_CLAC + + /* + * If the entry is from userspace, switch stacks and treat it as + * a normal entry. + */ + testb $3, CS-ORIG_RAX(%rsp) + jnz .Lfrom_usermode_switch_stack_\@ + + /* + * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX. + * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS + */ + call paranoid_entry + + UNWIND_HINT_REGS + + /* + * Switch off the IST stack to make it free for nested exceptions. The + * vc_switch_off_ist() function will switch back to the interrupted + * stack if it is safe to do so. If not it switches to the VC fall-back + * stack. + */ + movq %rsp, %rdi /* pt_regs pointer */ + call vc_switch_off_ist + movq %rax, %rsp /* Switch to new stack */ + + UNWIND_HINT_REGS + + /* Update pt_regs */ + movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/ + movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ + + movq %rsp, %rdi /* pt_regs pointer */ + + call \cfunc + + /* + * No need to switch back to the IST stack. The current stack is either + * identical to the stack in the IRET frame or the VC fall-back stack, + * so it is definitly mapped even with PTI enabled. + */ + jmp paranoid_exit + + /* Switch to the regular task stack */ +.Lfrom_usermode_switch_stack_\@: + idtentry_body safe_stack_\cfunc, has_error_code=1 + +_ASM_NOKPROBE(\asmsym) +SYM_CODE_END(\asmsym) +.endm +#endif + /* * Double fault entry. Straight paranoid. No checks from which context * this comes because for the espfix induced #DF this would do the wrong diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index a43366191212..840faaf57708 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -308,6 +308,18 @@ static __always_inline void __##func(struct pt_regs *regs) DECLARE_IDTENTRY_RAW(vector, func); \ __visible void noist_##func(struct pt_regs *regs) +/** + * DECLARE_IDTENTRY_VC - Declare functions for the VC entry point + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE, but declares also the + * safe_stack C handler. + */ +#define DECLARE_IDTENTRY_VC(vector, func) \ + DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \ + __visible noinstr void safe_stack_##func(struct pt_regs *regs, unsigned long error_code) + /** * DEFINE_IDTENTRY_IST - Emit code for IST entry points * @func: Function name of the entry point @@ -347,6 +359,35 @@ static __always_inline void __##func(struct pt_regs *regs) #define DEFINE_IDTENTRY_DF(func) \ DEFINE_IDTENTRY_RAW_ERRORCODE(func) +/** + * DEFINE_IDTENTRY_VC_SAFE_STACK - Emit code for VMM communication handler + which runs on a safe stack. + * @func: Function name of the entry point + * + * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE + */ +#define DEFINE_IDTENTRY_VC_SAFE_STACK(func) \ + DEFINE_IDTENTRY_RAW_ERRORCODE(safe_stack_##func) + +/** + * DEFINE_IDTENTRY_VC_IST - Emit code for VMM communication handler + which runs on the VC fall-back stack + * @func: Function name of the entry point + * + * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE + */ +#define DEFINE_IDTENTRY_VC_IST(func) \ + DEFINE_IDTENTRY_RAW_ERRORCODE(ist_##func) + +/** + * DEFINE_IDTENTRY_VC - Emit code for VMM communication handler + * @func: Function name of the entry point + * + * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE + */ +#define DEFINE_IDTENTRY_VC(func) \ + DEFINE_IDTENTRY_RAW_ERRORCODE(func) + #else /* CONFIG_X86_64 */ /** @@ -433,6 +474,9 @@ __visible noinstr void func(struct pt_regs *regs, \ # define DECLARE_IDTENTRY_XENCB(vector, func) \ DECLARE_IDTENTRY(vector, func) +# define DECLARE_IDTENTRY_VC(vector, func) \ + idtentry_vc vector asm_##func func + #else # define DECLARE_IDTENTRY_MCE(vector, func) \ DECLARE_IDTENTRY(vector, func) diff --git a/arch/x86/include/asm/proto.h b/arch/x86/include/asm/proto.h index 28996fe19301..2c35f1c01a2d 100644 --- a/arch/x86/include/asm/proto.h +++ b/arch/x86/include/asm/proto.h @@ -10,6 +10,7 @@ void syscall_init(void); #ifdef CONFIG_X86_64 void entry_SYSCALL_64(void); +void entry_SYSCALL_64_safe_stack(void); long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2); #endif diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index 6a308355ea29..1b86bb3abc56 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -15,6 +15,7 @@ asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs); asmlinkage __visible notrace struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s); void __init trap_init(void); +asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *eregs); #endif #ifdef CONFIG_X86_F00F_BUG diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 2dc32b0ca38e..fc1ed40ce97d 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -673,6 +674,50 @@ asmlinkage __visible noinstr struct pt_regs *sync_regs(struct pt_regs *eregs) return regs; } +#ifdef CONFIG_AMD_MEM_ENCRYPT +asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *regs) +{ + unsigned long sp, *stack; + struct stack_info info; + struct pt_regs *regs_ret; + + /* + * In the SYSCALL entry path the RSP value comes from user-space - don't + * trust it and switch to the current kernel stack + */ + if (regs->ip >= (unsigned long)entry_SYSCALL_64 && + regs->ip < (unsigned long)entry_SYSCALL_64_safe_stack) { + sp = this_cpu_read(cpu_current_top_of_stack); + goto sync; + } + + /* + * From here on the RSP value is trusted. Now check whether entry + * happened from a safe stack. Not safe are the entry or unknown stacks, + * use the fall-back stack instead in this case. + */ + sp = regs->sp; + stack = (unsigned long *)sp; + + if (!get_stack_info_noinstr(stack, current, &info) || info.type == STACK_TYPE_ENTRY || + info.type >= STACK_TYPE_EXCEPTION_LAST) + sp = __this_cpu_ist_top_va(VC2); + +sync: + /* + * Found a safe stack - switch to it as if the entry didn't happen via + * IST stack. The code below only copies pt_regs, the real switch happens + * in assembly code. + */ + sp = ALIGN_DOWN(sp, 8) - sizeof(*regs_ret); + + regs_ret = (struct pt_regs *)sp; + *regs_ret = *regs; + + return regs_ret; +} +#endif + struct bad_iret_stack { void *error_entry_ret; struct pt_regs regs; From 0786138c78e79343c7b015d77507cbf9d5f15d00 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:47 +0200 Subject: [PATCH 059/265] x86/sev-es: Add a Runtime #VC Exception Handler Add the handlers for #VC exceptions invoked at runtime. Signed-off-by: Tom Lendacky Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-47-joro@8bytes.org --- arch/x86/include/asm/idtentry.h | 6 + arch/x86/kernel/idt.c | 11 +- arch/x86/kernel/sev-es.c | 246 +++++++++++++++++++++++++++++++- 3 files changed, 255 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 840faaf57708..58a793b46a00 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -318,6 +318,7 @@ static __always_inline void __##func(struct pt_regs *regs) */ #define DECLARE_IDTENTRY_VC(vector, func) \ DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \ + __visible noinstr void ist_##func(struct pt_regs *regs, unsigned long error_code); \ __visible noinstr void safe_stack_##func(struct pt_regs *regs, unsigned long error_code) /** @@ -608,6 +609,11 @@ DECLARE_IDTENTRY_RAW(X86_TRAP_DB, xenpv_exc_debug); /* #DF */ DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_double_fault); +/* #VC */ +#ifdef CONFIG_AMD_MEM_ENCRYPT +DECLARE_IDTENTRY_VC(X86_TRAP_VC, exc_vmm_communication); +#endif + #ifdef CONFIG_XEN_PV DECLARE_IDTENTRY_XENCB(X86_TRAP_OTHER, exc_xen_hypervisor_callback); #endif diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index 4bb4e3d6099e..e29a6c728001 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -229,11 +229,14 @@ static const __initconst struct idt_data early_pf_idts[] = { * cpu_init() when the TSS has been initialized. */ static const __initconst struct idt_data ist_idts[] = { - ISTG(X86_TRAP_DB, asm_exc_debug, IST_INDEX_DB), - ISTG(X86_TRAP_NMI, asm_exc_nmi, IST_INDEX_NMI), - ISTG(X86_TRAP_DF, asm_exc_double_fault, IST_INDEX_DF), + ISTG(X86_TRAP_DB, asm_exc_debug, IST_INDEX_DB), + ISTG(X86_TRAP_NMI, asm_exc_nmi, IST_INDEX_NMI), + ISTG(X86_TRAP_DF, asm_exc_double_fault, IST_INDEX_DF), #ifdef CONFIG_X86_MCE - ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE), + ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE), +#endif +#ifdef CONFIG_AMD_MEM_ENCRYPT + ISTG(X86_TRAP_VC, asm_exc_vmm_communication, IST_INDEX_VC), #endif }; diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 39ebb2d04bec..0d6b66e93260 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -7,9 +7,12 @@ * Author: Joerg Roedel */ +#define pr_fmt(fmt) "SEV-ES: " fmt + #include /* For show_regs() */ #include #include +#include #include #include #include @@ -22,8 +25,8 @@ #include #include #include -#include -#include +#include +#include #include /* For early boot hypervisor communication in SEV-ES enabled guests */ @@ -48,11 +51,43 @@ struct sev_es_runtime_data { * interrupted stack in the #VC entry code. */ char fallback_stack[EXCEPTION_STKSZ] __aligned(PAGE_SIZE); + + /* + * Reserve one page per CPU as backup storage for the unencrypted GHCB. + * It is needed when an NMI happens while the #VC handler uses the real + * GHCB, and the NMI handler itself is causing another #VC exception. In + * that case the GHCB content of the first handler needs to be backed up + * and restored. + */ + struct ghcb backup_ghcb; + + /* + * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions. + * There is no need for it to be atomic, because nothing is written to + * the GHCB between the read and the write of ghcb_active. So it is safe + * to use it when a nested #VC exception happens before the write. + * + * This is necessary for example in the #VC->NMI->#VC case when the NMI + * happens while the first #VC handler uses the GHCB. When the NMI code + * raises a second #VC handler it might overwrite the contents of the + * GHCB written by the first handler. To avoid this the content of the + * GHCB is saved and restored when the GHCB is detected to be in use + * already. + */ + bool ghcb_active; + bool backup_ghcb_active; +}; + +struct ghcb_state { + struct ghcb *ghcb; }; static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); DEFINE_STATIC_KEY_FALSE(sev_es_enable_key); +/* Needed in vc_early_forward_exception */ +void do_early_exception(struct pt_regs *regs, int trapnr); + static void __init setup_vc_stacks(int cpu) { struct sev_es_runtime_data *data; @@ -123,8 +158,52 @@ void noinstr __sev_es_ist_exit(void) this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist); } -/* Needed in vc_early_forward_exception */ -void do_early_exception(struct pt_regs *regs, int trapnr); +static __always_inline struct ghcb *sev_es_get_ghcb(struct ghcb_state *state) +{ + struct sev_es_runtime_data *data; + struct ghcb *ghcb; + + data = this_cpu_read(runtime_data); + ghcb = &data->ghcb_page; + + if (unlikely(data->ghcb_active)) { + /* GHCB is already in use - save its contents */ + + if (unlikely(data->backup_ghcb_active)) + return NULL; + + /* Mark backup_ghcb active before writing to it */ + data->backup_ghcb_active = true; + + state->ghcb = &data->backup_ghcb; + + /* Backup GHCB content */ + *state->ghcb = *ghcb; + } else { + state->ghcb = NULL; + data->ghcb_active = true; + } + + return ghcb; +} + +static __always_inline void sev_es_put_ghcb(struct ghcb_state *state) +{ + struct sev_es_runtime_data *data; + struct ghcb *ghcb; + + data = this_cpu_read(runtime_data); + ghcb = &data->ghcb_page; + + if (state->ghcb) { + /* Restore GHCB from Backup */ + *ghcb = *state->ghcb; + data->backup_ghcb_active = false; + state->ghcb = NULL; + } else { + data->ghcb_active = false; + } +} static inline u64 sev_es_rd_ghcb_msr(void) { @@ -316,6 +395,9 @@ static void __init init_ghcb(int cpu) panic("Can't map GHCBs unencrypted"); memset(&data->ghcb_page, 0, sizeof(data->ghcb_page)); + + data->ghcb_active = false; + data->backup_ghcb_active = false; } void __init sev_es_init_vc_handling(void) @@ -336,6 +418,9 @@ void __init sev_es_init_vc_handling(void) init_ghcb(cpu); setup_vc_stacks(cpu); } + + /* Secondary CPUs use the runtime #VC handler */ + initial_vc_handler = (unsigned long)safe_stack_exc_vmm_communication; } static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) @@ -366,6 +451,159 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, return result; } +static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) +{ + long error_code = ctxt->fi.error_code; + int trapnr = ctxt->fi.vector; + + ctxt->regs->orig_ax = ctxt->fi.error_code; + + switch (trapnr) { + case X86_TRAP_GP: + exc_general_protection(ctxt->regs, error_code); + break; + case X86_TRAP_UD: + exc_invalid_op(ctxt->regs); + break; + default: + pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n"); + BUG(); + } +} + +static __always_inline bool on_vc_fallback_stack(struct pt_regs *regs) +{ + unsigned long sp = (unsigned long)regs; + + return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2)); +} + +/* + * Main #VC exception handler. It is called when the entry code was able to + * switch off the IST to a safe kernel stack. + * + * With the current implementation it is always possible to switch to a safe + * stack because #VC exceptions only happen at known places, like intercepted + * instructions or accesses to MMIO areas/IO ports. They can also happen with + * code instrumentation when the hypervisor intercepts #DB, but the critical + * paths are forbidden to be instrumented, so #DB exceptions currently also + * only happen in safe places. + */ +DEFINE_IDTENTRY_VC_SAFE_STACK(exc_vmm_communication) +{ + struct sev_es_runtime_data *data = this_cpu_read(runtime_data); + struct ghcb_state state; + struct es_em_ctxt ctxt; + enum es_result result; + struct ghcb *ghcb; + + lockdep_assert_irqs_disabled(); + instrumentation_begin(); + + /* + * This is invoked through an interrupt gate, so IRQs are disabled. The + * code below might walk page-tables for user or kernel addresses, so + * keep the IRQs disabled to protect us against concurrent TLB flushes. + */ + + ghcb = sev_es_get_ghcb(&state); + if (!ghcb) { + /* + * Mark GHCBs inactive so that panic() is able to print the + * message. + */ + data->ghcb_active = false; + data->backup_ghcb_active = false; + + panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use"); + } + + vc_ghcb_invalidate(ghcb); + result = vc_init_em_ctxt(&ctxt, regs, error_code); + + if (result == ES_OK) + result = vc_handle_exitcode(&ctxt, ghcb, error_code); + + sev_es_put_ghcb(&state); + + /* Done - now check the result */ + switch (result) { + case ES_OK: + vc_finish_insn(&ctxt); + break; + case ES_UNSUPPORTED: + pr_err_ratelimited("Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n", + error_code, regs->ip); + goto fail; + case ES_VMM_ERROR: + pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", + error_code, regs->ip); + goto fail; + case ES_DECODE_FAILED: + pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", + error_code, regs->ip); + goto fail; + case ES_EXCEPTION: + vc_forward_exception(&ctxt); + break; + case ES_RETRY: + /* Nothing to do */ + break; + default: + pr_emerg("Unknown result in %s():%d\n", __func__, result); + /* + * Emulating the instruction which caused the #VC exception + * failed - can't continue so print debug information + */ + BUG(); + } + +out: + instrumentation_end(); + + return; + +fail: + if (user_mode(regs)) { + /* + * Do not kill the machine if user-space triggered the + * exception. Send SIGBUS instead and let user-space deal with + * it. + */ + force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0); + } else { + pr_emerg("PANIC: Unhandled #VC exception in kernel space (result=%d)\n", + result); + + /* Show some debug info */ + show_regs(regs); + + /* Ask hypervisor to sev_es_terminate */ + sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); + + /* If that fails and we get here - just panic */ + panic("Returned from Terminate-Request to Hypervisor\n"); + } + + goto out; +} + +/* This handler runs on the #VC fall-back stack. It can cause further #VC exceptions */ +DEFINE_IDTENTRY_VC_IST(exc_vmm_communication) +{ + instrumentation_begin(); + panic("Can't handle #VC exception from unsupported context\n"); + instrumentation_end(); +} + +DEFINE_IDTENTRY_VC(exc_vmm_communication) +{ + if (likely(!on_vc_fallback_stack(regs))) + safe_stack_exc_vmm_communication(regs, error_code); + else + ist_exc_vmm_communication(regs, error_code); +} + bool __init handle_vc_boot_ghcb(struct pt_regs *regs) { unsigned long exit_code = regs->orig_ax; From d3529bb73f76d0ec8aafaca505226fa0971c1dc9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:48 +0200 Subject: [PATCH 060/265] x86/sev-es: Wire up existing #VC exit-code handlers Re-use the handlers for CPUID- and IOIO-caused #VC exceptions in the early boot handler. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-48-joro@8bytes.org --- arch/x86/kernel/sev-es-shared.c | 7 +++---- arch/x86/kernel/sev-es.c | 6 ++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 53e3de0e8060..491b557bdfba 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -325,8 +325,7 @@ static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) return ES_OK; } -static enum es_result __maybe_unused -vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; u64 exit_info_1, exit_info_2; @@ -434,8 +433,8 @@ vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) return ret; } -static enum es_result __maybe_unused vc_handle_cpuid(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) +static enum es_result vc_handle_cpuid(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; u32 cr4 = native_read_cr4(); diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 0d6b66e93260..b10a62a78c07 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -441,6 +441,12 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, enum es_result result; switch (exit_code) { + case SVM_EXIT_CPUID: + result = vc_handle_cpuid(ghcb, ctxt); + break; + case SVM_EXIT_IOIO: + result = vc_handle_ioio(ghcb, ctxt); + break; default: /* * Unexpected #VC exception From 5e3427a7bc432ed2e5de394ac30f160cc6c37a1f Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:49 +0200 Subject: [PATCH 061/265] x86/sev-es: Handle instruction fetches from user-space When a #VC exception is triggered by user-space, the instruction decoder needs to read the instruction bytes from user addresses. Enhance vc_decode_insn() to safely fetch kernel and user instructions. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-49-joro@8bytes.org --- arch/x86/kernel/sev-es.c | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index b10a62a78c07..6c30dbc1144a 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -232,16 +232,29 @@ static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) enum es_result ret; int res; - res = vc_fetch_insn_kernel(ctxt, buffer); - if (unlikely(res == -EFAULT)) { - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.error_code = 0; - ctxt->fi.cr2 = ctxt->regs->ip; - return ES_EXCEPTION; - } + if (user_mode(ctxt->regs)) { + res = insn_fetch_from_user(ctxt->regs, buffer); + if (!res) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER; + ctxt->fi.cr2 = ctxt->regs->ip; + return ES_EXCEPTION; + } - insn_init(&ctxt->insn, buffer, MAX_INSN_SIZE - res, 1); - insn_get_length(&ctxt->insn); + if (!insn_decode(&ctxt->insn, ctxt->regs, buffer, res)) + return ES_DECODE_FAILED; + } else { + res = vc_fetch_insn_kernel(ctxt, buffer); + if (res) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = X86_PF_INSTR; + ctxt->fi.cr2 = ctxt->regs->ip; + return ES_EXCEPTION; + } + + insn_init(&ctxt->insn, buffer, MAX_INSN_SIZE - res, 1); + insn_get_length(&ctxt->insn); + } ret = ctxt->insn.immediate.got ? ES_OK : ES_DECODE_FAILED; From 51ee7d6e3d2b70a3f232cceffab5084a2abd6719 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:50 +0200 Subject: [PATCH 062/265] x86/sev-es: Handle MMIO events Add a handler for #VC exceptions caused by MMIO intercepts. These intercepts come along as nested page faults on pages with reserved bits set. Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to VC handling framework ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-50-joro@8bytes.org --- arch/x86/include/uapi/asm/svm.h | 5 + arch/x86/kernel/sev-es.c | 222 ++++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index c68d1618c9b0..8f36ae021a7f 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -81,6 +81,11 @@ #define SVM_EXIT_AVIC_INCOMPLETE_IPI 0x401 #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS 0x402 +/* SEV-ES software-defined VMGEXIT events */ +#define SVM_VMGEXIT_MMIO_READ 0x80000001 +#define SVM_VMGEXIT_MMIO_WRITE 0x80000002 +#define SVM_VMGEXIT_UNSUPPORTED_EVENT 0x8000ffff + #define SVM_EXIT_ERR -1 #define SVM_EXIT_REASONS \ diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 6c30dbc1144a..a170810fe4f4 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -359,6 +359,37 @@ fault: return ES_EXCEPTION; } +static bool vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, + unsigned long vaddr, phys_addr_t *paddr) +{ + unsigned long va = (unsigned long)vaddr; + unsigned int level; + phys_addr_t pa; + pgd_t *pgd; + pte_t *pte; + + pgd = __va(read_cr3_pa()); + pgd = &pgd[pgd_index(va)]; + pte = lookup_address_in_pgd(pgd, va, &level); + if (!pte) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.cr2 = vaddr; + ctxt->fi.error_code = 0; + + if (user_mode(ctxt->regs)) + ctxt->fi.error_code |= X86_PF_USER; + + return false; + } + + pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT; + pa |= va & ~page_level_mask(level); + + *paddr = pa; + + return true; +} + /* Include code shared with pre-decompression boot stage */ #include "sev-es-shared.c" @@ -447,6 +478,194 @@ static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) do_early_exception(ctxt->regs, trapnr); } +static long *vc_insn_get_reg(struct es_em_ctxt *ctxt) +{ + long *reg_array; + int offset; + + reg_array = (long *)ctxt->regs; + offset = insn_get_modrm_reg_off(&ctxt->insn, ctxt->regs); + + if (offset < 0) + return NULL; + + offset /= sizeof(long); + + return reg_array + offset; +} + +static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, + unsigned int bytes, bool read) +{ + u64 exit_code, exit_info_1, exit_info_2; + unsigned long ghcb_pa = __pa(ghcb); + phys_addr_t paddr; + void __user *ref; + + ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs); + if (ref == (void __user *)-1L) + return ES_UNSUPPORTED; + + exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE; + + if (!vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr)) { + if (!read) + ctxt->fi.error_code |= X86_PF_WRITE; + + return ES_EXCEPTION; + } + + exit_info_1 = paddr; + /* Can never be greater than 8 */ + exit_info_2 = bytes; + + ghcb->save.sw_scratch = ghcb_pa + offsetof(struct ghcb, shared_buffer); + + return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2); +} + +static enum es_result vc_handle_mmio_twobyte_ops(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct insn *insn = &ctxt->insn; + unsigned int bytes = 0; + enum es_result ret; + int sign_byte; + long *reg_data; + + switch (insn->opcode.bytes[1]) { + /* MMIO Read w/ zero-extension */ + case 0xb6: + bytes = 1; + fallthrough; + case 0xb7: + if (!bytes) + bytes = 2; + + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + /* Zero extend based on operand size */ + reg_data = vc_insn_get_reg(ctxt); + if (!reg_data) + return ES_DECODE_FAILED; + + memset(reg_data, 0, insn->opnd_bytes); + + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + + /* MMIO Read w/ sign-extension */ + case 0xbe: + bytes = 1; + fallthrough; + case 0xbf: + if (!bytes) + bytes = 2; + + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + /* Sign extend based on operand size */ + reg_data = vc_insn_get_reg(ctxt); + if (!reg_data) + return ES_DECODE_FAILED; + + if (bytes == 1) { + u8 *val = (u8 *)ghcb->shared_buffer; + + sign_byte = (*val & 0x80) ? 0xff : 0x00; + } else { + u16 *val = (u16 *)ghcb->shared_buffer; + + sign_byte = (*val & 0x8000) ? 0xff : 0x00; + } + memset(reg_data, sign_byte, insn->opnd_bytes); + + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + + default: + ret = ES_UNSUPPORTED; + } + + return ret; +} + +static enum es_result vc_handle_mmio(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct insn *insn = &ctxt->insn; + unsigned int bytes = 0; + enum es_result ret; + long *reg_data; + + switch (insn->opcode.bytes[0]) { + /* MMIO Write */ + case 0x88: + bytes = 1; + fallthrough; + case 0x89: + if (!bytes) + bytes = insn->opnd_bytes; + + reg_data = vc_insn_get_reg(ctxt); + if (!reg_data) + return ES_DECODE_FAILED; + + memcpy(ghcb->shared_buffer, reg_data, bytes); + + ret = vc_do_mmio(ghcb, ctxt, bytes, false); + break; + + case 0xc6: + bytes = 1; + fallthrough; + case 0xc7: + if (!bytes) + bytes = insn->opnd_bytes; + + memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes); + + ret = vc_do_mmio(ghcb, ctxt, bytes, false); + break; + + /* MMIO Read */ + case 0x8a: + bytes = 1; + fallthrough; + case 0x8b: + if (!bytes) + bytes = insn->opnd_bytes; + + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + reg_data = vc_insn_get_reg(ctxt); + if (!reg_data) + return ES_DECODE_FAILED; + + /* Zero-extend for 32-bit operation */ + if (bytes == 4) + *reg_data = 0; + + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + + /* Two-Byte Opcodes */ + case 0x0f: + ret = vc_handle_mmio_twobyte_ops(ghcb, ctxt); + break; + default: + ret = ES_UNSUPPORTED; + } + + return ret; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -460,6 +679,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_IOIO: result = vc_handle_ioio(ghcb, ctxt); break; + case SVM_EXIT_NPF: + result = vc_handle_mmio(ghcb, ctxt); + break; default: /* * Unexpected #VC exception From 0118b604c2c94c6e34982015cfa7891af4764786 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:15:51 +0200 Subject: [PATCH 063/265] x86/sev-es: Handle MMIO String Instructions Add handling for emulation of the MOVS instruction on MMIO regions, as done by the memcpy_toio() and memcpy_fromio() functions. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-51-joro@8bytes.org --- arch/x86/kernel/sev-es.c | 77 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index a170810fe4f4..f724b75ad235 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -594,6 +594,73 @@ static enum es_result vc_handle_mmio_twobyte_ops(struct ghcb *ghcb, return ret; } +/* + * The MOVS instruction has two memory operands, which raises the + * problem that it is not known whether the access to the source or the + * destination caused the #VC exception (and hence whether an MMIO read + * or write operation needs to be emulated). + * + * Instead of playing games with walking page-tables and trying to guess + * whether the source or destination is an MMIO range, split the move + * into two operations, a read and a write with only one memory operand. + * This will cause a nested #VC exception on the MMIO address which can + * then be handled. + * + * This implementation has the benefit that it also supports MOVS where + * source _and_ destination are MMIO regions. + * + * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a + * rare operation. If it turns out to be a performance problem the split + * operations can be moved to memcpy_fromio() and memcpy_toio(). + */ +static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt, + unsigned int bytes) +{ + unsigned long ds_base, es_base; + unsigned char *src, *dst; + unsigned char buffer[8]; + enum es_result ret; + bool rep; + int off; + + ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS); + es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); + + if (ds_base == -1L || es_base == -1L) { + ctxt->fi.vector = X86_TRAP_GP; + ctxt->fi.error_code = 0; + return ES_EXCEPTION; + } + + src = ds_base + (unsigned char *)ctxt->regs->si; + dst = es_base + (unsigned char *)ctxt->regs->di; + + ret = vc_read_mem(ctxt, src, buffer, bytes); + if (ret != ES_OK) + return ret; + + ret = vc_write_mem(ctxt, dst, buffer, bytes); + if (ret != ES_OK) + return ret; + + if (ctxt->regs->flags & X86_EFLAGS_DF) + off = -bytes; + else + off = bytes; + + ctxt->regs->si += off; + ctxt->regs->di += off; + + rep = insn_has_rep_prefix(&ctxt->insn); + if (rep) + ctxt->regs->cx -= 1; + + if (!rep || ctxt->regs->cx == 0) + return ES_OK; + else + return ES_RETRY; +} + static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { @@ -655,6 +722,16 @@ static enum es_result vc_handle_mmio(struct ghcb *ghcb, memcpy(reg_data, ghcb->shared_buffer, bytes); break; + /* MOVS instruction */ + case 0xa4: + bytes = 1; + fallthrough; + case 0xa5: + if (!bytes) + bytes = insn->opnd_bytes; + + ret = vc_handle_mmio_movs(ctxt, bytes); + break; /* Two-Byte Opcodes */ case 0x0f: ret = vc_handle_mmio_twobyte_ops(ghcb, ctxt); From a4afa6081c88701635e1e20090f953a25f9444e0 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:52 +0200 Subject: [PATCH 064/265] x86/sev-es: Handle MSR events Implement a handler for #VC exceptions caused by RDMSR/WRMSR instructions. Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling infrastructure. ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-52-joro@8bytes.org --- arch/x86/kernel/sev-es.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index f724b75ad235..620510070a1a 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -393,6 +393,31 @@ static bool vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, /* Include code shared with pre-decompression boot stage */ #include "sev-es-shared.c" +static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + struct pt_regs *regs = ctxt->regs; + enum es_result ret; + u64 exit_info_1; + + /* Is it a WRMSR? */ + exit_info_1 = (ctxt->insn.opcode.bytes[1] == 0x30) ? 1 : 0; + + ghcb_set_rcx(ghcb, regs->cx); + if (exit_info_1) { + ghcb_set_rax(ghcb, regs->ax); + ghcb_set_rdx(ghcb, regs->dx); + } + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, exit_info_1, 0); + + if ((ret == ES_OK) && (!exit_info_1)) { + regs->ax = ghcb->save.rax; + regs->dx = ghcb->save.rdx; + } + + return ret; +} + /* * This function runs on the first #VC exception after the kernel * switched to virtual addresses. @@ -756,6 +781,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_IOIO: result = vc_handle_ioio(ghcb, ctxt); break; + case SVM_EXIT_MSR: + result = vc_handle_msr(ghcb, ctxt); + break; case SVM_EXIT_NPF: result = vc_handle_mmio(ghcb, ctxt); break; From 479a7bf5c1f7f14eb37c6d6aeb401562efc1fcad Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:53 +0200 Subject: [PATCH 065/265] x86/sev-es: Handle DR7 read/write events Add code to handle #VC exceptions on DR7 register reads and writes. This is needed early because show_regs() reads DR7 to print it out. Under SEV-ES, there is currently no support for saving/restoring the DRx registers but software expects to be able to write to the DR7 register. For now, cache the value written to DR7 and return it on read attempts, but do not touch the real hardware DR7. Signed-off-by: Tom Lendacky [ jroedel@suse.de: - Adapt to #VC handling framework - Support early usage ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-53-joro@8bytes.org --- arch/x86/kernel/sev-es.c | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 620510070a1a..391b9c7f7312 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -29,6 +29,8 @@ #include #include +#define DR7_RESET_VALUE 0x400 + /* For early boot hypervisor communication in SEV-ES enabled guests */ static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); @@ -76,6 +78,13 @@ struct sev_es_runtime_data { */ bool ghcb_active; bool backup_ghcb_active; + + /* + * Cached DR7 value - write it on DR7 writes and return it on reads. + * That value will never make it to the real hardware DR7 as debugging + * is currently unsupported in SEV-ES guests. + */ + unsigned long dr7; }; struct ghcb_state { @@ -519,6 +528,21 @@ static long *vc_insn_get_reg(struct es_em_ctxt *ctxt) return reg_array + offset; } +static long *vc_insn_get_rm(struct es_em_ctxt *ctxt) +{ + long *reg_array; + int offset; + + reg_array = (long *)ctxt->regs; + offset = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs); + + if (offset < 0) + return NULL; + + offset /= sizeof(long); + + return reg_array + offset; +} static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, unsigned int bytes, bool read) { @@ -768,6 +792,61 @@ static enum es_result vc_handle_mmio(struct ghcb *ghcb, return ret; } +static enum es_result vc_handle_dr7_write(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct sev_es_runtime_data *data = this_cpu_read(runtime_data); + long val, *reg = vc_insn_get_rm(ctxt); + enum es_result ret; + + if (!reg) + return ES_DECODE_FAILED; + + val = *reg; + + /* Upper 32 bits must be written as zeroes */ + if (val >> 32) { + ctxt->fi.vector = X86_TRAP_GP; + ctxt->fi.error_code = 0; + return ES_EXCEPTION; + } + + /* Clear out other reserved bits and set bit 10 */ + val = (val & 0xffff23ffL) | BIT(10); + + /* Early non-zero writes to DR7 are not supported */ + if (!data && (val & ~DR7_RESET_VALUE)) + return ES_UNSUPPORTED; + + /* Using a value of 0 for ExitInfo1 means RAX holds the value */ + ghcb_set_rax(ghcb, val); + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0); + if (ret != ES_OK) + return ret; + + if (data) + data->dr7 = val; + + return ES_OK; +} + +static enum es_result vc_handle_dr7_read(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct sev_es_runtime_data *data = this_cpu_read(runtime_data); + long *reg = vc_insn_get_rm(ctxt); + + if (!reg) + return ES_DECODE_FAILED; + + if (data) + *reg = data->dr7; + else + *reg = DR7_RESET_VALUE; + + return ES_OK; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -775,6 +854,12 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, enum es_result result; switch (exit_code) { + case SVM_EXIT_READ_DR7: + result = vc_handle_dr7_read(ghcb, ctxt); + break; + case SVM_EXIT_WRITE_DR7: + result = vc_handle_dr7_write(ghcb, ctxt); + break; case SVM_EXIT_CPUID: result = vc_handle_cpuid(ghcb, ctxt); break; From a14a92fc4b420ae6713800a8ae8c3686c1a1fa20 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:54 +0200 Subject: [PATCH 066/265] x86/sev-es: Handle WBINVD Events Implement a handler for #VC exceptions caused by WBINVD instructions. Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling framework ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-54-joro@8bytes.org --- arch/x86/kernel/sev-es.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 391b9c7f7312..aba27c3c1633 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -847,6 +847,12 @@ static enum es_result vc_handle_dr7_read(struct ghcb *ghcb, return ES_OK; } +static enum es_result vc_handle_wbinvd(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0); +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -869,6 +875,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_MSR: result = vc_handle_msr(ghcb, ctxt); break; + case SVM_EXIT_WBINVD: + result = vc_handle_wbinvd(ghcb, ctxt); + break; case SVM_EXIT_NPF: result = vc_handle_mmio(ghcb, ctxt); break; From 4711e7acaa125d8cc242f06e1f4d6c74e177454b Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:55 +0200 Subject: [PATCH 067/265] x86/sev-es: Handle RDTSC(P) Events Implement a handler for #VC exceptions caused by RDTSC and RDTSCP instructions. Also make it available in the pre-decompression stage because the KASLR code uses RDTSC/RDTSCP to gather entropy and some hypervisors intercept these instructions. Signed-off-by: Tom Lendacky [ jroedel@suse.de: - Adapt to #VC handling infrastructure - Make it available early ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-55-joro@8bytes.org --- arch/x86/boot/compressed/sev-es.c | 4 ++++ arch/x86/kernel/sev-es-shared.c | 23 +++++++++++++++++++++++ arch/x86/kernel/sev-es.c | 4 ++++ 3 files changed, 31 insertions(+) diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index b1790f487456..5f15e5864e0c 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -181,6 +181,10 @@ void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) goto finish; switch (exit_code) { + case SVM_EXIT_RDTSC: + case SVM_EXIT_RDTSCP: + result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code); + break; case SVM_EXIT_IOIO: result = vc_handle_ioio(boot_ghcb, &ctxt); break; diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 491b557bdfba..4be8af2f9c57 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -467,3 +467,26 @@ static enum es_result vc_handle_cpuid(struct ghcb *ghcb, return ES_OK; } + +static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, + struct es_em_ctxt *ctxt, + unsigned long exit_code) +{ + bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); + enum es_result ret; + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); + if (ret != ES_OK) + return ret; + + if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && + (!rdtscp || ghcb_rcx_is_valid(ghcb)))) + return ES_VMM_ERROR; + + ctxt->regs->ax = ghcb->save.rax; + ctxt->regs->dx = ghcb->save.rdx; + if (rdtscp) + ctxt->regs->cx = ghcb->save.rcx; + + return ES_OK; +} diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index aba27c3c1633..4d468ec325c3 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -866,6 +866,10 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_WRITE_DR7: result = vc_handle_dr7_write(ghcb, ctxt); break; + case SVM_EXIT_RDTSC: + case SVM_EXIT_RDTSCP: + result = vc_handle_rdtsc(ghcb, ctxt, exit_code); + break; case SVM_EXIT_CPUID: result = vc_handle_cpuid(ghcb, ctxt); break; From 5d55cf78a8785a7496919462f3d1a4689f62fb1d Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:56 +0200 Subject: [PATCH 068/265] x86/sev-es: Handle RDPMC Events Implement a handler for #VC exceptions caused by RDPMC instructions. Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling infrastructure ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-56-joro@8bytes.org --- arch/x86/kernel/sev-es.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 4d468ec325c3..b2d7287a9975 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -853,6 +853,25 @@ static enum es_result vc_handle_wbinvd(struct ghcb *ghcb, return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0); } +static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + enum es_result ret; + + ghcb_set_rcx(ghcb, ctxt->regs->cx); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0); + if (ret != ES_OK) + return ret; + + if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb))) + return ES_VMM_ERROR; + + ctxt->regs->ax = ghcb->save.rax; + ctxt->regs->dx = ghcb->save.rdx; + + return ES_OK; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -870,6 +889,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_RDTSCP: result = vc_handle_rdtsc(ghcb, ctxt, exit_code); break; + case SVM_EXIT_RDPMC: + result = vc_handle_rdpmc(ghcb, ctxt); + break; case SVM_EXIT_CPUID: result = vc_handle_cpuid(ghcb, ctxt); break; From 8b4ce83707cb2cff5f6d175ea38f751ac4456096 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:57 +0200 Subject: [PATCH 069/265] x86/sev-es: Handle INVD Events Implement a handler for #VC exceptions caused by INVD instructions. Since Linux should never use INVD, just mark it as unsupported. Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling infrastructure ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-57-joro@8bytes.org --- arch/x86/kernel/sev-es.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index b2d7287a9975..236cfc16f696 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -892,6 +892,10 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_RDPMC: result = vc_handle_rdpmc(ghcb, ctxt); break; + case SVM_EXIT_INVD: + pr_err_ratelimited("#VC exception for INVD??? Seriously???\n"); + result = ES_UNSUPPORTED; + break; case SVM_EXIT_CPUID: result = vc_handle_cpuid(ghcb, ctxt); break; From 0c2fd2ef64ef1a91d81c2f61309735ac438b68a4 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:58 +0200 Subject: [PATCH 070/265] x86/sev-es: Handle MONITOR/MONITORX Events Implement a handler for #VC exceptions caused by MONITOR and MONITORX instructions. Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling infrastructure ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-58-joro@8bytes.org --- arch/x86/kernel/sev-es.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 236cfc16f696..b9976e710d77 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -872,6 +872,16 @@ static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt return ES_OK; } +static enum es_result vc_handle_monitor(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + /* + * Treat it as a NOP and do not leak a physical address to the + * hypervisor. + */ + return ES_OK; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -908,6 +918,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_WBINVD: result = vc_handle_wbinvd(ghcb, ctxt); break; + case SVM_EXIT_MONITOR: + result = vc_handle_monitor(ghcb, ctxt); + break; case SVM_EXIT_NPF: result = vc_handle_mmio(ghcb, ctxt); break; From ded476bbe203e59cb571d6c576f680efe4a0ddff Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:15:59 +0200 Subject: [PATCH 071/265] x86/sev-es: Handle MWAIT/MWAITX Events Implement a handler for #VC exceptions caused by MWAIT and MWAITX instructions. Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling infrastructure ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-59-joro@8bytes.org --- arch/x86/kernel/sev-es.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index b9976e710d77..2aea903ceb42 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -882,6 +882,13 @@ static enum es_result vc_handle_monitor(struct ghcb *ghcb, return ES_OK; } +static enum es_result vc_handle_mwait(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + /* Treat the same as MONITOR/MONITORX */ + return ES_OK; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -921,6 +928,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_MONITOR: result = vc_handle_monitor(ghcb, ctxt); break; + case SVM_EXIT_MWAIT: + result = vc_handle_mwait(ghcb, ctxt); + break; case SVM_EXIT_NPF: result = vc_handle_mmio(ghcb, ctxt); break; From 2eb7dcf0ccc40ad3f39b000becf16661abf98102 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:16:00 +0200 Subject: [PATCH 072/265] x86/sev-es: Handle VMMCALL Events Implement a handler for #VC exceptions caused by VMMCALL instructions. This is only a starting point, VMMCALL emulation under SEV-ES needs further hypervisor-specific changes to provide additional state. [ bp: Drop "this patch". ] Signed-off-by: Tom Lendacky [ jroedel@suse.de: Adapt to #VC handling infrastructure ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-60-joro@8bytes.org --- arch/x86/kernel/sev-es.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 2aea903ceb42..86cb4c5a7ad2 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -889,6 +889,26 @@ static enum es_result vc_handle_mwait(struct ghcb *ghcb, return ES_OK; } +static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + enum es_result ret; + + ghcb_set_rax(ghcb, ctxt->regs->ax); + ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0); + if (ret != ES_OK) + return ret; + + if (!ghcb_rax_is_valid(ghcb)) + return ES_VMM_ERROR; + + ctxt->regs->ax = ghcb->save.rax; + + return ES_OK; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -922,6 +942,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_MSR: result = vc_handle_msr(ghcb, ctxt); break; + case SVM_EXIT_VMMCALL: + result = vc_handle_vmmcall(ghcb, ctxt); + break; case SVM_EXIT_WBINVD: result = vc_handle_wbinvd(ghcb, ctxt); break; From a2d0171a9cf59637411281a929900fde80e6c1cb Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:01 +0200 Subject: [PATCH 073/265] x86/sev-es: Handle #AC Events Implement a handler for #VC exceptions caused by #AC exceptions. The #AC exception is just forwarded to do_alignment_check() and not pushed down to the hypervisor, as requested by the SEV-ES GHCB Standardization Specification. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-61-joro@8bytes.org --- arch/x86/kernel/sev-es.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 86cb4c5a7ad2..8867c481e3e2 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -909,6 +909,19 @@ static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, return ES_OK; } +static enum es_result vc_handle_trap_ac(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + /* + * Calling ecx_alignment_check() directly does not work, because it + * enables IRQs and the GHCB is active. Forward the exception and call + * it later from vc_forward_exception(). + */ + ctxt->fi.vector = X86_TRAP_AC; + ctxt->fi.error_code = 0; + return ES_EXCEPTION; +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -922,6 +935,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, case SVM_EXIT_WRITE_DR7: result = vc_handle_dr7_write(ghcb, ctxt); break; + case SVM_EXIT_EXCP_BASE + X86_TRAP_AC: + result = vc_handle_trap_ac(ghcb, ctxt); + break; case SVM_EXIT_RDTSC: case SVM_EXIT_RDTSCP: result = vc_handle_rdtsc(ghcb, ctxt, exit_code); @@ -981,6 +997,9 @@ static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) case X86_TRAP_UD: exc_invalid_op(ctxt->regs); break; + case X86_TRAP_AC: + exc_alignment_check(ctxt->regs, error_code); + break; default: pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n"); BUG(); From cb1ad3ecea959593400dfac4f027dbc005e62c39 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:02 +0200 Subject: [PATCH 074/265] x86/sev-es: Handle #DB Events Handle #VC exceptions caused by #DB exceptions in the guest. Those must be handled outside of instrumentation_begin()/end() so that the handler will not be raised recursively. Handle them by calling the kernel's debug exception handler. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-62-joro@8bytes.org --- arch/x86/kernel/sev-es.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 8867c481e3e2..79d5190cbc6b 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -922,6 +922,14 @@ static enum es_result vc_handle_trap_ac(struct ghcb *ghcb, return ES_EXCEPTION; } +static __always_inline void vc_handle_trap_db(struct pt_regs *regs) +{ + if (user_mode(regs)) + noist_exc_debug(regs); + else + exc_debug(regs); +} + static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, struct ghcb *ghcb, unsigned long exit_code) @@ -1033,6 +1041,15 @@ DEFINE_IDTENTRY_VC_SAFE_STACK(exc_vmm_communication) struct ghcb *ghcb; lockdep_assert_irqs_disabled(); + + /* + * Handle #DB before calling into !noinstr code to avoid recursive #DB. + */ + if (error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB) { + vc_handle_trap_db(regs); + return; + } + instrumentation_begin(); /* From f6a9f8a45810d2914ea422ff39bfe2e0251c50f2 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:03 +0200 Subject: [PATCH 075/265] x86/paravirt: Allow hypervisor-specific VMMCALL handling under SEV-ES Add two new paravirt callbacks to provide hypervisor-specific processor state in the GHCB and to copy state from the hypervisor back to the processor. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-63-joro@8bytes.org --- arch/x86/include/asm/x86_init.h | 16 +++++++++++++++- arch/x86/kernel/sev-es.c | 12 ++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h index 6807153c0410..0304e2931cd3 100644 --- a/arch/x86/include/asm/x86_init.h +++ b/arch/x86/include/asm/x86_init.h @@ -4,8 +4,10 @@ #include +struct ghcb; struct mpc_bus; struct mpc_cpu; +struct pt_regs; struct mpc_table; struct cpuinfo_x86; @@ -236,10 +238,22 @@ struct x86_legacy_features { /** * struct x86_hyper_runtime - x86 hypervisor specific runtime callbacks * - * @pin_vcpu: pin current vcpu to specified physical cpu (run rarely) + * @pin_vcpu: pin current vcpu to specified physical + * cpu (run rarely) + * @sev_es_hcall_prepare: Load additional hypervisor-specific + * state into the GHCB when doing a VMMCALL under + * SEV-ES. Called from the #VC exception handler. + * @sev_es_hcall_finish: Copies state from the GHCB back into the + * processor (or pt_regs). Also runs checks on the + * state returned from the hypervisor after a + * VMMCALL under SEV-ES. Needs to return 'false' + * if the checks fail. Called from the #VC + * exception handler. */ struct x86_hyper_runtime { void (*pin_vcpu)(int cpu); + void (*sev_es_hcall_prepare)(struct ghcb *ghcb, struct pt_regs *regs); + bool (*sev_es_hcall_finish)(struct ghcb *ghcb, struct pt_regs *regs); }; /** diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 79d5190cbc6b..6d89df9f29dc 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -897,6 +897,9 @@ static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, ghcb_set_rax(ghcb, ctxt->regs->ax); ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0); + if (x86_platform.hyper.sev_es_hcall_prepare) + x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs); + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0); if (ret != ES_OK) return ret; @@ -906,6 +909,15 @@ static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, ctxt->regs->ax = ghcb->save.rax; + /* + * Call sev_es_hcall_finish() after regs->ax is already set. + * This allows the hypervisor handler to overwrite it again if + * necessary. + */ + if (x86_platform.hyper.sev_es_hcall_finish && + !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs)) + return ES_VMM_ERROR; + return ES_OK; } From 99419b251e5427b89dbfae103d8a2f469efaa4b2 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:16:04 +0200 Subject: [PATCH 076/265] x86/kvm: Add KVM-specific VMMCALL handling under SEV-ES Implement the callbacks to copy the processor state required by KVM to the GHCB. Signed-off-by: Tom Lendacky [ jroedel@suse.de: - Split out of a larger patch - Adapt to different callback functions ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-64-joro@8bytes.org --- arch/x86/kernel/kvm.c | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 08320b0b2b27..0f9597275e9c 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include DEFINE_STATIC_KEY_FALSE(kvm_async_pf_enabled); @@ -746,13 +748,34 @@ static void __init kvm_init_platform(void) x86_platform.apic_post_init = kvm_apic_init; } +#if defined(CONFIG_AMD_MEM_ENCRYPT) +static void kvm_sev_es_hcall_prepare(struct ghcb *ghcb, struct pt_regs *regs) +{ + /* RAX and CPL are already in the GHCB */ + ghcb_set_rbx(ghcb, regs->bx); + ghcb_set_rcx(ghcb, regs->cx); + ghcb_set_rdx(ghcb, regs->dx); + ghcb_set_rsi(ghcb, regs->si); +} + +static bool kvm_sev_es_hcall_finish(struct ghcb *ghcb, struct pt_regs *regs) +{ + /* No checking of the return state needed */ + return true; +} +#endif + const __initconst struct hypervisor_x86 x86_hyper_kvm = { - .name = "KVM", - .detect = kvm_detect, - .type = X86_HYPER_KVM, - .init.guest_late_init = kvm_guest_init, - .init.x2apic_available = kvm_para_available, - .init.init_platform = kvm_init_platform, + .name = "KVM", + .detect = kvm_detect, + .type = X86_HYPER_KVM, + .init.guest_late_init = kvm_guest_init, + .init.x2apic_available = kvm_para_available, + .init.init_platform = kvm_init_platform, +#if defined(CONFIG_AMD_MEM_ENCRYPT) + .runtime.sev_es_hcall_prepare = kvm_sev_es_hcall_prepare, + .runtime.sev_es_hcall_finish = kvm_sev_es_hcall_finish, +#endif }; static __init int activate_jump_labels(void) From 1a222de8dcfb903d039810b0823570ee0be4e6c6 Mon Sep 17 00:00:00 2001 From: Doug Covelli Date: Mon, 7 Sep 2020 15:16:05 +0200 Subject: [PATCH 077/265] x86/vmware: Add VMware-specific handling for VMMCALL under SEV-ES Add VMware-specific handling for #VC faults caused by VMMCALL instructions. Signed-off-by: Doug Covelli Signed-off-by: Tom Lendacky [ jroedel@suse.de: - Adapt to different paravirt interface ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-65-joro@8bytes.org --- arch/x86/kernel/cpu/vmware.c | 50 ++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index 9b6fafa69be9..924571fe5864 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -33,6 +33,7 @@ #include #include #include +#include #undef pr_fmt #define pr_fmt(fmt) "vmware: " fmt @@ -476,10 +477,49 @@ static bool __init vmware_legacy_x2apic_available(void) (eax & (1 << VMWARE_CMD_LEGACY_X2APIC)) != 0; } +#ifdef CONFIG_AMD_MEM_ENCRYPT +static void vmware_sev_es_hcall_prepare(struct ghcb *ghcb, + struct pt_regs *regs) +{ + /* Copy VMWARE specific Hypercall parameters to the GHCB */ + ghcb_set_rip(ghcb, regs->ip); + ghcb_set_rbx(ghcb, regs->bx); + ghcb_set_rcx(ghcb, regs->cx); + ghcb_set_rdx(ghcb, regs->dx); + ghcb_set_rsi(ghcb, regs->si); + ghcb_set_rdi(ghcb, regs->di); + ghcb_set_rbp(ghcb, regs->bp); +} + +static bool vmware_sev_es_hcall_finish(struct ghcb *ghcb, struct pt_regs *regs) +{ + if (!(ghcb_rbx_is_valid(ghcb) && + ghcb_rcx_is_valid(ghcb) && + ghcb_rdx_is_valid(ghcb) && + ghcb_rsi_is_valid(ghcb) && + ghcb_rdi_is_valid(ghcb) && + ghcb_rbp_is_valid(ghcb))) + return false; + + regs->bx = ghcb->save.rbx; + regs->cx = ghcb->save.rcx; + regs->dx = ghcb->save.rdx; + regs->si = ghcb->save.rsi; + regs->di = ghcb->save.rdi; + regs->bp = ghcb->save.rbp; + + return true; +} +#endif + const __initconst struct hypervisor_x86 x86_hyper_vmware = { - .name = "VMware", - .detect = vmware_platform, - .type = X86_HYPER_VMWARE, - .init.init_platform = vmware_platform_setup, - .init.x2apic_available = vmware_legacy_x2apic_available, + .name = "VMware", + .detect = vmware_platform, + .type = X86_HYPER_VMWARE, + .init.init_platform = vmware_platform_setup, + .init.x2apic_available = vmware_legacy_x2apic_available, +#ifdef CONFIG_AMD_MEM_ENCRYPT + .runtime.sev_es_hcall_prepare = vmware_sev_es_hcall_prepare, + .runtime.sev_es_hcall_finish = vmware_sev_es_hcall_finish, +#endif }; From bf5ff276448f64f1f9ef9ffc9e231026e3887d3d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:06 +0200 Subject: [PATCH 078/265] x86/realmode: Add SEV-ES specific trampoline entry point The code at the trampoline entry point is executed in real-mode. In real-mode, #VC exceptions can't be handled so anything that might cause such an exception must be avoided. In the standard trampoline entry code this is the WBINVD instruction and the call to verify_cpu(), which are both not needed anyway when running as an SEV-ES guest. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-66-joro@8bytes.org --- arch/x86/include/asm/realmode.h | 3 +++ arch/x86/realmode/rm/header.S | 3 +++ arch/x86/realmode/rm/trampoline_64.S | 20 ++++++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/arch/x86/include/asm/realmode.h b/arch/x86/include/asm/realmode.h index 96118fb041b8..4d4d853f6841 100644 --- a/arch/x86/include/asm/realmode.h +++ b/arch/x86/include/asm/realmode.h @@ -21,6 +21,9 @@ struct real_mode_header { /* SMP trampoline */ u32 trampoline_start; u32 trampoline_header; +#ifdef CONFIG_AMD_MEM_ENCRYPT + u32 sev_es_trampoline_start; +#endif #ifdef CONFIG_X86_64 u32 trampoline_pgd; #endif diff --git a/arch/x86/realmode/rm/header.S b/arch/x86/realmode/rm/header.S index af04512c02d9..8c1db5bf5d78 100644 --- a/arch/x86/realmode/rm/header.S +++ b/arch/x86/realmode/rm/header.S @@ -20,6 +20,9 @@ SYM_DATA_START(real_mode_header) /* SMP trampoline */ .long pa_trampoline_start .long pa_trampoline_header +#ifdef CONFIG_AMD_MEM_ENCRYPT + .long pa_sev_es_trampoline_start +#endif #ifdef CONFIG_X86_64 .long pa_trampoline_pgd; #endif diff --git a/arch/x86/realmode/rm/trampoline_64.S b/arch/x86/realmode/rm/trampoline_64.S index 251758ed7443..84c5d1b33d10 100644 --- a/arch/x86/realmode/rm/trampoline_64.S +++ b/arch/x86/realmode/rm/trampoline_64.S @@ -56,6 +56,7 @@ SYM_CODE_START(trampoline_start) testl %eax, %eax # Check for return code jnz no_longmode +.Lswitch_to_protected: /* * GDT tables in non default location kernel can be beyond 16MB and * lgdt will not be able to load the address as in real mode default @@ -80,6 +81,25 @@ no_longmode: jmp no_longmode SYM_CODE_END(trampoline_start) +#ifdef CONFIG_AMD_MEM_ENCRYPT +/* SEV-ES supports non-zero IP for entry points - no alignment needed */ +SYM_CODE_START(sev_es_trampoline_start) + cli # We should be safe anyway + + LJMPW_RM(1f) +1: + mov %cs, %ax # Code and data in the same place + mov %ax, %ds + mov %ax, %es + mov %ax, %ss + + # Setup stack + movl $rm_stack_end, %esp + + jmp .Lswitch_to_protected +SYM_CODE_END(sev_es_trampoline_start) +#endif /* CONFIG_AMD_MEM_ENCRYPT */ + #include "../kernel/verify_cpu.S" .section ".text32","ax" From 8940ac9ced8bc1c48c4e28b0784e3234c9d14469 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:16:07 +0200 Subject: [PATCH 079/265] x86/realmode: Setup AP jump table As part of the GHCB specification, the booting of APs under SEV-ES requires an AP jump table when transitioning from one layer of code to another (e.g. when going from UEFI to the OS). As a result, each layer that parks an AP must provide the physical address of an AP jump table to the next layer via the hypervisor. Upon booting of the kernel, read the AP jump table address from the hypervisor. Under SEV-ES, APs are started using the INIT-SIPI-SIPI sequence. Before issuing the first SIPI request for an AP, the start CS and IP is programmed into the AP jump table. Upon issuing the SIPI request, the AP will awaken and jump to that start CS:IP address. Signed-off-by: Tom Lendacky [ jroedel@suse.de: - Adapted to different code base - Moved AP table setup from SIPI sending path to real-mode setup code - Fix sparse warnings ] Co-developed-by: Joerg Roedel Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-67-joro@8bytes.org --- arch/x86/include/asm/sev-es.h | 5 +++ arch/x86/include/uapi/asm/svm.h | 3 ++ arch/x86/kernel/sev-es.c | 69 +++++++++++++++++++++++++++++++++ arch/x86/realmode/init.c | 18 ++++++++- 4 files changed, 93 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index 59176e8c6b81..db88e1c3442d 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -73,6 +73,9 @@ static inline u64 lower_bits(u64 val, unsigned int bits) return (val & mask); } +struct real_mode_header; +enum stack_type; + /* Early IDT entry points for #VC handler */ extern void vc_no_ghcb(void); extern void vc_boot_ghcb(void); @@ -92,9 +95,11 @@ static __always_inline void sev_es_ist_exit(void) if (static_branch_unlikely(&sev_es_enable_key)) __sev_es_ist_exit(); } +extern int sev_es_setup_ap_jump_table(struct real_mode_header *rmh); #else static inline void sev_es_ist_enter(struct pt_regs *regs) { } static inline void sev_es_ist_exit(void) { } +static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) { return 0; } #endif #endif diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index 8f36ae021a7f..346b8a7155e8 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -84,6 +84,9 @@ /* SEV-ES software-defined VMGEXIT events */ #define SVM_VMGEXIT_MMIO_READ 0x80000001 #define SVM_VMGEXIT_MMIO_WRITE 0x80000002 +#define SVM_VMGEXIT_AP_JUMP_TABLE 0x80000005 +#define SVM_VMGEXIT_SET_AP_JUMP_TABLE 0 +#define SVM_VMGEXIT_GET_AP_JUMP_TABLE 1 #define SVM_VMGEXIT_UNSUPPORTED_EVENT 0x8000ffff #define SVM_EXIT_ERR -1 diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 6d89df9f29dc..9ad36ce92b82 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -214,6 +215,9 @@ static __always_inline void sev_es_put_ghcb(struct ghcb_state *state) } } +/* Needed in vc_early_forward_exception */ +void do_early_exception(struct pt_regs *regs, int trapnr); + static inline u64 sev_es_rd_ghcb_msr(void) { return __rdmsr(MSR_AMD64_SEV_ES_GHCB); @@ -402,6 +406,71 @@ static bool vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, /* Include code shared with pre-decompression boot stage */ #include "sev-es-shared.c" +static u64 get_jump_table_addr(void) +{ + struct ghcb_state state; + unsigned long flags; + struct ghcb *ghcb; + u64 ret = 0; + + local_irq_save(flags); + + ghcb = sev_es_get_ghcb(&state); + + vc_ghcb_invalidate(ghcb); + ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE); + ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE); + ghcb_set_sw_exit_info_2(ghcb, 0); + + sev_es_wr_ghcb_msr(__pa(ghcb)); + VMGEXIT(); + + if (ghcb_sw_exit_info_1_is_valid(ghcb) && + ghcb_sw_exit_info_2_is_valid(ghcb)) + ret = ghcb->save.sw_exit_info_2; + + sev_es_put_ghcb(&state); + + local_irq_restore(flags); + + return ret; +} + +int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) +{ + u16 startup_cs, startup_ip; + phys_addr_t jump_table_pa; + u64 jump_table_addr; + u16 __iomem *jump_table; + + jump_table_addr = get_jump_table_addr(); + + /* On UP guests there is no jump table so this is not a failure */ + if (!jump_table_addr) + return 0; + + /* Check if AP Jump Table is page-aligned */ + if (jump_table_addr & ~PAGE_MASK) + return -EINVAL; + + jump_table_pa = jump_table_addr & PAGE_MASK; + + startup_cs = (u16)(rmh->trampoline_start >> 4); + startup_ip = (u16)(rmh->sev_es_trampoline_start - + rmh->trampoline_start); + + jump_table = ioremap_encrypted(jump_table_pa, PAGE_SIZE); + if (!jump_table) + return -EIO; + + writew(startup_ip, &jump_table[0]); + writew(startup_cs, &jump_table[1]); + + iounmap(jump_table); + + return 0; +} + static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c index 1ed1208931e0..3fb9b60be07a 100644 --- a/arch/x86/realmode/init.c +++ b/arch/x86/realmode/init.c @@ -9,6 +9,7 @@ #include #include #include +#include struct real_mode_header *real_mode_header; u32 *trampoline_cr4_features; @@ -38,6 +39,19 @@ void __init reserve_real_mode(void) crash_reserve_low_1M(); } +static void sme_sev_setup_real_mode(struct trampoline_header *th) +{ +#ifdef CONFIG_AMD_MEM_ENCRYPT + if (sme_active()) + th->flags |= TH_FLAGS_SME_ACTIVE; + + if (sev_es_active()) { + if (sev_es_setup_ap_jump_table(real_mode_header)) + panic("Failed to get/update SEV-ES AP Jump Table"); + } +#endif +} + static void __init setup_real_mode(void) { u16 real_mode_seg; @@ -104,13 +118,13 @@ static void __init setup_real_mode(void) *trampoline_cr4_features = mmu_cr4_features; trampoline_header->flags = 0; - if (sme_active()) - trampoline_header->flags |= TH_FLAGS_SME_ACTIVE; trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd); trampoline_pgd[0] = trampoline_pgd_entry.pgd; trampoline_pgd[511] = init_top_pgt[511].pgd; #endif + + sme_sev_setup_real_mode(trampoline_header); } /* From 520d030852b4c9babfce9a79d8b5320b6b5545e6 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:08 +0200 Subject: [PATCH 080/265] x86/smpboot: Load TSS and getcpu GDT entry before loading IDT The IDT on 64-bit contains vectors which use paranoid_entry() and/or IST stacks. To make these vectors work, the TSS and the getcpu GDT entry need to be set up before the IDT is loaded. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-68-joro@8bytes.org --- arch/x86/include/asm/processor.h | 1 + arch/x86/kernel/cpu/common.c | 23 +++++++++++++++++++++++ arch/x86/kernel/smpboot.c | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 97143d87994c..615dd440bed8 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -696,6 +696,7 @@ extern void load_direct_gdt(int); extern void load_fixmap_gdt(int); extern void load_percpu_segment(int); extern void cpu_init(void); +extern void cpu_init_exception_handling(void); extern void cr4_init(void); static inline unsigned long get_debugctlmsr(void) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 81fba4d4a242..beffea22e2af 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1862,6 +1862,29 @@ static inline void tss_setup_io_bitmap(struct tss_struct *tss) #endif } +/* + * Setup everything needed to handle exceptions from the IDT, including the IST + * exceptions which use paranoid_entry(). + */ +void cpu_init_exception_handling(void) +{ + struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw); + int cpu = raw_smp_processor_id(); + + /* paranoid_entry() gets the CPU number from the GDT */ + setup_getcpu(cpu); + + /* IST vectors need TSS to be set up. */ + tss_setup_ist(tss); + tss_setup_io_bitmap(tss); + set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss); + + load_TR_desc(); + + /* Finally load the IDT */ + load_current_idt(); +} + /* * cpu_init() initializes state that is per-CPU. Some data is already * initialized (naturally) in the bootstrap process, such as the GDT diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index f5ef689dd62a..de776b2e6046 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -227,7 +227,7 @@ static void notrace start_secondary(void *unused) load_cr3(swapper_pg_dir); __flush_tlb_all(); #endif - load_current_idt(); + cpu_init_exception_handling(); cpu_init(); x86_cpuinit.early_percpu_clock_init(); preempt_disable(); From 3ecacdbd23956a549d93023f86adc87b4a9d6520 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:09 +0200 Subject: [PATCH 081/265] x86/head/64: Don't call verify_cpu() on starting APs The APs are not ready to handle exceptions when verify_cpu() is called in secondary_startup_64(). Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-69-joro@8bytes.org --- arch/x86/include/asm/realmode.h | 1 + arch/x86/kernel/head_64.S | 12 ++++++++++++ arch/x86/realmode/init.c | 6 ++++++ 3 files changed, 19 insertions(+) diff --git a/arch/x86/include/asm/realmode.h b/arch/x86/include/asm/realmode.h index 4d4d853f6841..5db5d083c873 100644 --- a/arch/x86/include/asm/realmode.h +++ b/arch/x86/include/asm/realmode.h @@ -72,6 +72,7 @@ extern unsigned char startup_32_smp[]; extern unsigned char boot_gdt[]; #else extern unsigned char secondary_startup_64[]; +extern unsigned char secondary_startup_64_no_verify[]; #endif static inline size_t real_mode_size_needed(void) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 1a71d0d4d575..7eb2a1c87969 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -125,6 +125,18 @@ SYM_CODE_START(secondary_startup_64) /* Sanitize CPU configuration */ call verify_cpu + /* + * The secondary_startup_64_no_verify entry point is only used by + * SEV-ES guests. In those guests the call to verify_cpu() would cause + * #VC exceptions which can not be handled at this stage of secondary + * CPU bringup. + * + * All non SEV-ES systems, especially Intel systems, need to execute + * verify_cpu() above to make sure NX is enabled. + */ +SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) + UNWIND_HINT_EMPTY + /* * Retrieve the modifier (SME encryption mask if SME is active) to be * added to the initial pgdir entry that will be programmed into CR3. diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c index 3fb9b60be07a..22fda7d99159 100644 --- a/arch/x86/realmode/init.c +++ b/arch/x86/realmode/init.c @@ -46,6 +46,12 @@ static void sme_sev_setup_real_mode(struct trampoline_header *th) th->flags |= TH_FLAGS_SME_ACTIVE; if (sev_es_active()) { + /* + * Skip the call to verify_cpu() in secondary_startup_64 as it + * will cause #VC exceptions when the AP can't handle them yet. + */ + th->start = (u64) secondary_startup_64_no_verify; + if (sev_es_setup_ap_jump_table(real_mode_header)) panic("Failed to get/update SEV-ES AP Jump Table"); } From 094794f59720d7e877a1eeb372ecedeed6b441ab Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:10 +0200 Subject: [PATCH 082/265] x86/sev-es: Support CPU offline/online Add a play_dead handler when running under SEV-ES. This is needed because the hypervisor can't deliver an SIPI request to restart the AP. Instead, the kernel has to issue a VMGEXIT to halt the VCPU until the hypervisor wakes it up again. Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-70-joro@8bytes.org --- arch/x86/include/uapi/asm/svm.h | 1 + arch/x86/kernel/sev-es.c | 63 +++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index 346b8a7155e8..c1dcf3e114e4 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -84,6 +84,7 @@ /* SEV-ES software-defined VMGEXIT events */ #define SVM_VMGEXIT_MMIO_READ 0x80000001 #define SVM_VMGEXIT_MMIO_WRITE 0x80000002 +#define SVM_VMGEXIT_AP_HLT_LOOP 0x80000004 #define SVM_VMGEXIT_AP_JUMP_TABLE 0x80000005 #define SVM_VMGEXIT_SET_AP_JUMP_TABLE 0 #define SVM_VMGEXIT_GET_AP_JUMP_TABLE 1 diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 9ad36ce92b82..d1bcd0dca583 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -29,6 +29,8 @@ #include #include #include +#include +#include #define DR7_RESET_VALUE 0x400 @@ -518,6 +520,65 @@ static bool __init sev_es_setup_ghcb(void) return true; } +#ifdef CONFIG_HOTPLUG_CPU +static void sev_es_ap_hlt_loop(void) +{ + struct ghcb_state state; + struct ghcb *ghcb; + + ghcb = sev_es_get_ghcb(&state); + + while (true) { + vc_ghcb_invalidate(ghcb); + ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_HLT_LOOP); + ghcb_set_sw_exit_info_1(ghcb, 0); + ghcb_set_sw_exit_info_2(ghcb, 0); + + sev_es_wr_ghcb_msr(__pa(ghcb)); + VMGEXIT(); + + /* Wakeup signal? */ + if (ghcb_sw_exit_info_2_is_valid(ghcb) && + ghcb->save.sw_exit_info_2) + break; + } + + sev_es_put_ghcb(&state); +} + +/* + * Play_dead handler when running under SEV-ES. This is needed because + * the hypervisor can't deliver an SIPI request to restart the AP. + * Instead the kernel has to issue a VMGEXIT to halt the VCPU until the + * hypervisor wakes it up again. + */ +static void sev_es_play_dead(void) +{ + play_dead_common(); + + /* IRQs now disabled */ + + sev_es_ap_hlt_loop(); + + /* + * If we get here, the VCPU was woken up again. Jump to CPU + * startup code to get it back online. + */ + start_cpu0(); +} +#else /* CONFIG_HOTPLUG_CPU */ +#define sev_es_play_dead native_play_dead +#endif /* CONFIG_HOTPLUG_CPU */ + +#ifdef CONFIG_SMP +static void __init sev_es_setup_play_dead(void) +{ + smp_ops.play_dead = sev_es_play_dead; +} +#else +static inline void sev_es_setup_play_dead(void) { } +#endif + static void __init alloc_runtime_data(int cpu) { struct sev_es_runtime_data *data; @@ -566,6 +627,8 @@ void __init sev_es_init_vc_handling(void) setup_vc_stacks(cpu); } + sev_es_setup_play_dead(); + /* Secondary CPUs use the runtime #VC handler */ initial_vc_handler = (unsigned long)safe_stack_exc_vmm_communication; } From 4ca68e023b11e4d5908bf9ee326fab01111d77d5 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 7 Sep 2020 15:16:11 +0200 Subject: [PATCH 083/265] x86/sev-es: Handle NMI State When running under SEV-ES, the kernel has to tell the hypervisor when to open the NMI window again after an NMI was injected. This is done with an NMI-complete message to the hypervisor. Add code to the kernel's NMI handler to send this message right at the beginning of do_nmi(). This always allows nesting NMIs. [ bp: Mark __sev_es_nmi_complete() noinstr: vmlinux.o: warning: objtool: exc_nmi()+0x17: call to __sev_es_nmi_complete() leaves .noinstr.text section While at it, use __pa_nodebug() for the same reason due to CONFIG_DEBUG_VIRTUAL=y: vmlinux.o: warning: objtool: __sev_es_nmi_complete()+0xd9: call to __phys_addr() leaves .noinstr.text section ] Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20200907131613.12703-71-joro@8bytes.org --- arch/x86/include/asm/sev-es.h | 7 +++++++ arch/x86/include/uapi/asm/svm.h | 1 + arch/x86/kernel/nmi.c | 6 ++++++ arch/x86/kernel/sev-es.c | 18 ++++++++++++++++++ 4 files changed, 32 insertions(+) diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index db88e1c3442d..e919f09ae33c 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -96,10 +96,17 @@ static __always_inline void sev_es_ist_exit(void) __sev_es_ist_exit(); } extern int sev_es_setup_ap_jump_table(struct real_mode_header *rmh); +extern void __sev_es_nmi_complete(void); +static __always_inline void sev_es_nmi_complete(void) +{ + if (static_branch_unlikely(&sev_es_enable_key)) + __sev_es_nmi_complete(); +} #else static inline void sev_es_ist_enter(struct pt_regs *regs) { } static inline void sev_es_ist_exit(void) { } static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) { return 0; } +static inline void sev_es_nmi_complete(void) { } #endif #endif diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index c1dcf3e114e4..a7a3403645e5 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -84,6 +84,7 @@ /* SEV-ES software-defined VMGEXIT events */ #define SVM_VMGEXIT_MMIO_READ 0x80000001 #define SVM_VMGEXIT_MMIO_WRITE 0x80000002 +#define SVM_VMGEXIT_NMI_COMPLETE 0x80000003 #define SVM_VMGEXIT_AP_HLT_LOOP 0x80000004 #define SVM_VMGEXIT_AP_JUMP_TABLE 0x80000005 #define SVM_VMGEXIT_SET_AP_JUMP_TABLE 0 diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 4c89c4d3cb82..56b64d779856 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -478,6 +478,12 @@ DEFINE_IDTENTRY_RAW(exc_nmi) { bool irq_state; + /* + * Re-enable NMIs right here when running as an SEV-ES guest. This might + * cause nested NMIs, but those can be handled safely. + */ + sev_es_nmi_complete(); + if (IS_ENABLED(CONFIG_SMP) && arch_cpu_is_offline(smp_processor_id())) return; diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index d1bcd0dca583..b6518e96dedb 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -408,6 +408,24 @@ static bool vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, /* Include code shared with pre-decompression boot stage */ #include "sev-es-shared.c" +void noinstr __sev_es_nmi_complete(void) +{ + struct ghcb_state state; + struct ghcb *ghcb; + + ghcb = sev_es_get_ghcb(&state); + + vc_ghcb_invalidate(ghcb); + ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE); + ghcb_set_sw_exit_info_1(ghcb, 0); + ghcb_set_sw_exit_info_2(ghcb, 0); + + sev_es_wr_ghcb_msr(__pa_nodebug(ghcb)); + VMGEXIT(); + + sev_es_put_ghcb(&state); +} + static u64 get_jump_table_addr(void) { struct ghcb_state state; From ded10c47f39e0f850d53f5a2b418fbafdef53347 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 20 Aug 2020 15:46:20 +0530 Subject: [PATCH 084/265] cpufreq: imx6q: Unconditionally call dev_pm_opp_of_remove_table() dev_pm_opp_of_remove_table() doesn't report any errors when it fails to find the OPP table with error -ENODEV (i.e. OPP table not present for the device). And we can call dev_pm_opp_of_remove_table() unconditionally here. Signed-off-by: Viresh Kumar --- drivers/cpufreq/imx6q-cpufreq.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index ef7b34c1fd2b..5bf5fc759881 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -48,7 +48,6 @@ static struct clk_bulk_data clks[] = { }; static struct device *cpu_dev; -static bool free_opp; static struct cpufreq_frequency_table *freq_table; static unsigned int max_freq; static unsigned int transition_latency; @@ -390,9 +389,6 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) goto put_reg; } - /* Because we have added the OPPs here, we must free them */ - free_opp = true; - if (of_machine_is_compatible("fsl,imx6ul") || of_machine_is_compatible("fsl,imx6ull")) { ret = imx6ul_opp_check_speed_grading(cpu_dev); @@ -507,8 +503,7 @@ soc_opp_out: free_freq_table: dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); out_free_opp: - if (free_opp) - dev_pm_opp_of_remove_table(cpu_dev); + dev_pm_opp_of_remove_table(cpu_dev); put_reg: if (!IS_ERR(arm_reg)) regulator_put(arm_reg); @@ -528,8 +523,7 @@ static int imx6q_cpufreq_remove(struct platform_device *pdev) { cpufreq_unregister_driver(&imx6q_cpufreq_driver); dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); - if (free_opp) - dev_pm_opp_of_remove_table(cpu_dev); + dev_pm_opp_of_remove_table(cpu_dev); regulator_put(arm_reg); if (!IS_ERR(pu_reg)) regulator_put(pu_reg); From 4db7b6aacc573b89227a1b5960af9d2a17f17762 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 2 Sep 2020 10:31:03 -0700 Subject: [PATCH 085/265] swiotlb: Use %pa to print phys_addr_t variables There is an extension to a %p to print phys_addr_t type of variables. Use it here. Signed-off-by: Andy Shevchenko Reviewed-by: Fabio Estevam Signed-off-by: Konrad Rzeszutek Wilk --- kernel/dma/swiotlb.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index c19379fabd20..6499bda8f0b8 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -172,9 +172,7 @@ void swiotlb_print_info(void) return; } - pr_info("mapped [mem %#010llx-%#010llx] (%luMB)\n", - (unsigned long long)io_tlb_start, - (unsigned long long)io_tlb_end, + pr_info("mapped [mem %pa-%pa] (%luMB)\n", &io_tlb_start, &io_tlb_end, bytes >> 20); } From 61b82bbf693ecd307550ee64a8af192e8e33c46c Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 2 Sep 2020 20:31:04 +0300 Subject: [PATCH 086/265] swiotlb: Declare swiotlb_late_init_with_default_size() in header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compiler is not happy about one function prototype: CC kernel/dma/swiotlb.o kernel/dma/swiotlb.c:275:1: warning: no previous prototype for ‘swiotlb_late_init_with_default_size’ [-Wmissing-prototypes] 275 | swiotlb_late_init_with_default_size(size_t default_size) | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Since it's used outside of the module, move its declaration to the header from the user. Signed-off-by: Andy Shevchenko Signed-off-by: Konrad Rzeszutek Wilk --- arch/x86/pci/sta2x11-fixup.c | 1 - include/linux/swiotlb.h | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/pci/sta2x11-fixup.c b/arch/x86/pci/sta2x11-fixup.c index c313d784efab..11c0e80b9ed4 100644 --- a/arch/x86/pci/sta2x11-fixup.c +++ b/arch/x86/pci/sta2x11-fixup.c @@ -15,7 +15,6 @@ #include #define STA2X11_SWIOTLB_SIZE (4*1024*1024) -extern int swiotlb_late_init_with_default_size(size_t default_size); /* * We build a list of bus numbers that are under the ConneXt. The diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 046bb94bd4d6..513913ff7486 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -34,6 +34,7 @@ int swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose); extern unsigned long swiotlb_nr_tbl(void); unsigned long swiotlb_size_or_default(void); extern int swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs); +extern int swiotlb_late_init_with_default_size(size_t default_size); extern void __init swiotlb_update_mem_attributes(void); /* From b51e627158cb0f67569aa46b5a08b568d20a81a4 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 2 Sep 2020 20:31:05 +0300 Subject: [PATCH 087/265] swiotlb: Mark max_segment with static keyword Sparse is not happy about max_segment declaration: CHECK kernel/dma/swiotlb.c kernel/dma/swiotlb.c:96:14: warning: symbol 'max_segment' was not declared. Should it be static? Mark it static as suggested. Signed-off-by: Andy Shevchenko Signed-off-by: Konrad Rzeszutek Wilk --- kernel/dma/swiotlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 6499bda8f0b8..465a567678d9 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -93,7 +93,7 @@ static unsigned int io_tlb_index; * Max segment that we can provide which (if pages are contingous) will * not be bounced (unless SWIOTLB_FORCE is set). */ -unsigned int max_segment; +static unsigned int max_segment; /* * We need to save away the original address corresponding to a mapped entry From 3890b8d92710af75baedf291832cf40193b33454 Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Fri, 4 Sep 2020 16:30:19 +0100 Subject: [PATCH 088/265] objtool: Group headers to check in a single list In order to support multiple architectures and potentially different sets of headers to compare against their kernel equivalent, it is simpler to have all headers to check in a single list. Reviewed-by: Miroslav Benes Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- tools/objtool/sync-check.sh | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tools/objtool/sync-check.sh b/tools/objtool/sync-check.sh index aa099b21dffa..b5f526638f0a 100755 --- a/tools/objtool/sync-check.sh +++ b/tools/objtool/sync-check.sh @@ -1,14 +1,18 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0 -FILES=' +FILES=" arch/x86/include/asm/inat_types.h arch/x86/include/asm/orc_types.h arch/x86/include/asm/emulate_prefix.h arch/x86/lib/x86-opcode-map.txt arch/x86/tools/gen-insn-attr-x86.awk include/linux/static_call_types.h -' +arch/x86/include/asm/inat.h -I '^#include [\"<]\(asm/\)*inat_types.h[\">]' +arch/x86/include/asm/insn.h -I '^#include [\"<]\(asm/\)*inat.h[\">]' +arch/x86/lib/inat.c -I '^#include [\"<]\(../include/\)*asm/insn.h[\">]' +arch/x86/lib/insn.c -I '^#include [\"<]\(../include/\)*asm/in\(at\|sn\).h[\">]' -I '^#include [\"<]\(../include/\)*asm/emulate_prefix.h[\">]' +" check_2 () { file1=$1 @@ -41,11 +45,12 @@ fi cd ../.. -for i in $FILES; do - check $i -done +while read -r file_entry; do + if [ -z "$file_entry" ]; then + continue + fi -check arch/x86/include/asm/inat.h '-I "^#include [\"<]\(asm/\)*inat_types.h[\">]"' -check arch/x86/include/asm/insn.h '-I "^#include [\"<]\(asm/\)*inat.h[\">]"' -check arch/x86/lib/inat.c '-I "^#include [\"<]\(../include/\)*asm/insn.h[\">]"' -check arch/x86/lib/insn.c '-I "^#include [\"<]\(../include/\)*asm/in\(at\|sn\).h[\">]" -I "^#include [\"<]\(../include/\)*asm/emulate_prefix.h[\">]"' + check $file_entry +done < Date: Fri, 4 Sep 2020 16:30:20 +0100 Subject: [PATCH 089/265] objtool: Make sync-check consider the target architecture Do not take into account outdated headers unrelated to the build of the current architecture. [ jpoimboe: use $SRCARCH directly ] Reviewed-by: Miroslav Benes Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- tools/objtool/sync-check.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/objtool/sync-check.sh b/tools/objtool/sync-check.sh index b5f526638f0a..cea1c12607b9 100755 --- a/tools/objtool/sync-check.sh +++ b/tools/objtool/sync-check.sh @@ -1,6 +1,12 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0 +if [ -z "$SRCARCH" ]; then + echo 'sync-check.sh: error: missing $SRCARCH environment variable' >&2 + exit 1 +fi + +if [ "$SRCARCH" = "x86" ]; then FILES=" arch/x86/include/asm/inat_types.h arch/x86/include/asm/orc_types.h @@ -13,6 +19,7 @@ arch/x86/include/asm/insn.h -I '^#include [\"<]\(asm/\)*inat.h[\">]' arch/x86/lib/inat.c -I '^#include [\"<]\(../include/\)*asm/insn.h[\">]' arch/x86/lib/insn.c -I '^#include [\"<]\(../include/\)*asm/in\(at\|sn\).h[\">]' -I '^#include [\"<]\(../include/\)*asm/emulate_prefix.h[\">]' " +fi check_2 () { file1=$1 From c8ea0d672521ef663f0f9a77faa94d0d47102d77 Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Fri, 4 Sep 2020 16:30:21 +0100 Subject: [PATCH 090/265] objtool: Move macros describing structures to arch-dependent code Some macros are defined to describe the size and layout of structures exception_table_entry, jump_entry and alt_instr. These values can vary from one architecture to another. Have the values be defined by arch specific code. Suggested-by: Raphael Gault Reviewed-by: Miroslav Benes Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- tools/objtool/arch/x86/include/arch_special.h | 20 +++++++++++++++++++ tools/objtool/special.c | 16 +-------------- 2 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 tools/objtool/arch/x86/include/arch_special.h diff --git a/tools/objtool/arch/x86/include/arch_special.h b/tools/objtool/arch/x86/include/arch_special.h new file mode 100644 index 000000000000..d818b2bffa02 --- /dev/null +++ b/tools/objtool/arch/x86/include/arch_special.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _X86_ARCH_SPECIAL_H +#define _X86_ARCH_SPECIAL_H + +#define EX_ENTRY_SIZE 12 +#define EX_ORIG_OFFSET 0 +#define EX_NEW_OFFSET 4 + +#define JUMP_ENTRY_SIZE 16 +#define JUMP_ORIG_OFFSET 0 +#define JUMP_NEW_OFFSET 4 + +#define ALT_ENTRY_SIZE 13 +#define ALT_ORIG_OFFSET 0 +#define ALT_NEW_OFFSET 4 +#define ALT_FEATURE_OFFSET 8 +#define ALT_ORIG_LEN_OFFSET 10 +#define ALT_NEW_LEN_OFFSET 11 + +#endif /* _X86_ARCH_SPECIAL_H */ diff --git a/tools/objtool/special.c b/tools/objtool/special.c index e893f1e48e44..b04f395de5de 100644 --- a/tools/objtool/special.c +++ b/tools/objtool/special.c @@ -14,21 +14,7 @@ #include "builtin.h" #include "special.h" #include "warn.h" - -#define EX_ENTRY_SIZE 12 -#define EX_ORIG_OFFSET 0 -#define EX_NEW_OFFSET 4 - -#define JUMP_ENTRY_SIZE 16 -#define JUMP_ORIG_OFFSET 0 -#define JUMP_NEW_OFFSET 4 - -#define ALT_ENTRY_SIZE 13 -#define ALT_ORIG_OFFSET 0 -#define ALT_NEW_OFFSET 4 -#define ALT_FEATURE_OFFSET 8 -#define ALT_ORIG_LEN_OFFSET 10 -#define ALT_NEW_LEN_OFFSET 11 +#include "arch_special.h" #define X86_FEATURE_POPCNT (4*32+23) #define X86_FEATURE_SMAP (9*32+20) From eda3dc905834dc9c99132f987f77b68cf53a8682 Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Fri, 4 Sep 2020 16:30:22 +0100 Subject: [PATCH 091/265] objtool: Abstract alternative special case handling Some alternatives associated with a specific feature need to be treated in a special way. Since the features and how to treat them vary from one architecture to another, move the special case handling to arch specific code. Reviewed-by: Miroslav Benes Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- tools/objtool/arch/x86/Build | 1 + tools/objtool/arch/x86/special.c | 37 ++++++++++++++++++++++++++++++++ tools/objtool/objtool.h | 2 ++ tools/objtool/special.c | 32 +++++---------------------- tools/objtool/special.h | 2 ++ tools/objtool/weak.c | 2 -- 6 files changed, 47 insertions(+), 29 deletions(-) create mode 100644 tools/objtool/arch/x86/special.c diff --git a/tools/objtool/arch/x86/Build b/tools/objtool/arch/x86/Build index 7c5004008e97..9f7869b5c5e0 100644 --- a/tools/objtool/arch/x86/Build +++ b/tools/objtool/arch/x86/Build @@ -1,3 +1,4 @@ +objtool-y += special.o objtool-y += decode.o inat_tables_script = ../arch/x86/tools/gen-insn-attr-x86.awk diff --git a/tools/objtool/arch/x86/special.c b/tools/objtool/arch/x86/special.c new file mode 100644 index 000000000000..823561e4015c --- /dev/null +++ b/tools/objtool/arch/x86/special.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include "../../special.h" +#include "../../builtin.h" + +#define X86_FEATURE_POPCNT (4 * 32 + 23) +#define X86_FEATURE_SMAP (9 * 32 + 20) + +void arch_handle_alternative(unsigned short feature, struct special_alt *alt) +{ + switch (feature) { + case X86_FEATURE_SMAP: + /* + * If UACCESS validation is enabled; force that alternative; + * otherwise force it the other way. + * + * What we want to avoid is having both the original and the + * alternative code flow at the same time, in that case we can + * find paths that see the STAC but take the NOP instead of + * CLAC and the other way around. + */ + if (uaccess) + alt->skip_orig = true; + else + alt->skip_alt = true; + break; + case X86_FEATURE_POPCNT: + /* + * It has been requested that we don't validate the !POPCNT + * feature path which is a "very very small percentage of + * machines". + */ + alt->skip_orig = true; + break; + default: + break; + } +} diff --git a/tools/objtool/objtool.h b/tools/objtool/objtool.h index a635f68a9b09..4125d4578b23 100644 --- a/tools/objtool/objtool.h +++ b/tools/objtool/objtool.h @@ -12,6 +12,8 @@ #include "elf.h" +#define __weak __attribute__((weak)) + struct objtool_file { struct elf *elf; struct list_head insn_list; diff --git a/tools/objtool/special.c b/tools/objtool/special.c index b04f395de5de..1a2420febd08 100644 --- a/tools/objtool/special.c +++ b/tools/objtool/special.c @@ -16,9 +16,6 @@ #include "warn.h" #include "arch_special.h" -#define X86_FEATURE_POPCNT (4*32+23) -#define X86_FEATURE_SMAP (9*32+20) - struct special_entry { const char *sec; bool group, jump_or_nop; @@ -54,6 +51,10 @@ struct special_entry entries[] = { {}, }; +void __weak arch_handle_alternative(unsigned short feature, struct special_alt *alt) +{ +} + static int get_alt_entry(struct elf *elf, struct special_entry *entry, struct section *sec, int idx, struct special_alt *alt) @@ -78,30 +79,7 @@ static int get_alt_entry(struct elf *elf, struct special_entry *entry, feature = *(unsigned short *)(sec->data->d_buf + offset + entry->feature); - - /* - * It has been requested that we don't validate the !POPCNT - * feature path which is a "very very small percentage of - * machines". - */ - if (feature == X86_FEATURE_POPCNT) - alt->skip_orig = true; - - /* - * If UACCESS validation is enabled; force that alternative; - * otherwise force it the other way. - * - * What we want to avoid is having both the original and the - * alternative code flow at the same time, in that case we can - * find paths that see the STAC but take the NOP instead of - * CLAC and the other way around. - */ - if (feature == X86_FEATURE_SMAP) { - if (uaccess) - alt->skip_orig = true; - else - alt->skip_alt = true; - } + arch_handle_alternative(feature, alt); } orig_reloc = find_reloc_by_dest(elf, sec, offset + entry->orig); diff --git a/tools/objtool/special.h b/tools/objtool/special.h index 35061530e46e..44da89afeda2 100644 --- a/tools/objtool/special.h +++ b/tools/objtool/special.h @@ -28,4 +28,6 @@ struct special_alt { int special_get_alts(struct elf *elf, struct list_head *alts); +void arch_handle_alternative(unsigned short feature, struct special_alt *alt); + #endif /* _SPECIAL_H */ diff --git a/tools/objtool/weak.c b/tools/objtool/weak.c index 29180d599b08..7843e9a7a72f 100644 --- a/tools/objtool/weak.c +++ b/tools/objtool/weak.c @@ -9,8 +9,6 @@ #include #include "objtool.h" -#define __weak __attribute__((weak)) - #define UNSUPPORTED(name) \ ({ \ fprintf(stderr, "error: objtool: " name " not implemented\n"); \ From 45245f51f9a4b9a883a8c94468473c1de9b88153 Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Fri, 4 Sep 2020 16:30:23 +0100 Subject: [PATCH 092/265] objtool: Make relocation in alternative handling arch dependent As pointed out by the comment in handle_group_alt(), support of relocation for instructions in an alternative group depends on whether arch specific kernel code handles it. So, let objtool arch specific code decide whether a relocation for the alternative section should be accepted. Reviewed-by: Miroslav Benes Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- tools/objtool/arch/x86/special.c | 13 +++++++++++++ tools/objtool/check.c | 19 ++++++------------- tools/objtool/check.h | 6 ++++++ tools/objtool/special.h | 4 ++++ 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/tools/objtool/arch/x86/special.c b/tools/objtool/arch/x86/special.c index 823561e4015c..34e0e162e6fd 100644 --- a/tools/objtool/arch/x86/special.c +++ b/tools/objtool/arch/x86/special.c @@ -35,3 +35,16 @@ void arch_handle_alternative(unsigned short feature, struct special_alt *alt) break; } } + +bool arch_support_alt_relocation(struct special_alt *special_alt, + struct instruction *insn, + struct reloc *reloc) +{ + /* + * The x86 alternatives code adjusts the offsets only when it + * encounters a branch instruction at the very beginning of the + * replacement group. + */ + return insn->offset == special_alt->new_off && + (insn->type == INSN_CALL || is_static_jump(insn)); +} diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 4afc2d5465b9..1796a7c464eb 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -110,12 +110,6 @@ static struct instruction *prev_insn_same_sym(struct objtool_file *file, for (insn = next_insn_same_sec(file, insn); insn; \ insn = next_insn_same_sec(file, insn)) -static bool is_static_jump(struct instruction *insn) -{ - return insn->type == INSN_JUMP_CONDITIONAL || - insn->type == INSN_JUMP_UNCONDITIONAL; -} - static bool is_sibling_call(struct instruction *insn) { /* An indirect jump is either a sibling call or a jump to a table. */ @@ -972,6 +966,8 @@ static int handle_group_alt(struct objtool_file *file, alt_group = alt_group_next_index++; insn = *new_insn; sec_for_each_insn_from(file, insn) { + struct reloc *alt_reloc; + if (insn->offset >= special_alt->new_off + special_alt->new_len) break; @@ -988,14 +984,11 @@ static int handle_group_alt(struct objtool_file *file, * .altinstr_replacement section, unless the arch's * alternatives code can adjust the relative offsets * accordingly. - * - * The x86 alternatives code adjusts the offsets only when it - * encounters a branch instruction at the very beginning of the - * replacement group. */ - if ((insn->offset != special_alt->new_off || - (insn->type != INSN_CALL && !is_static_jump(insn))) && - find_reloc_by_dest_range(file->elf, insn->sec, insn->offset, insn->len)) { + alt_reloc = find_reloc_by_dest_range(file->elf, insn->sec, + insn->offset, insn->len); + if (alt_reloc && + !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { WARN_FUNC("unsupported relocation in alternatives section", insn->sec, insn->offset); diff --git a/tools/objtool/check.h b/tools/objtool/check.h index 6cac34542122..1de1188b23cd 100644 --- a/tools/objtool/check.h +++ b/tools/objtool/check.h @@ -48,6 +48,12 @@ struct instruction { #endif }; +static inline bool is_static_jump(struct instruction *insn) +{ + return insn->type == INSN_JUMP_CONDITIONAL || + insn->type == INSN_JUMP_UNCONDITIONAL; +} + struct instruction *find_insn(struct objtool_file *file, struct section *sec, unsigned long offset); diff --git a/tools/objtool/special.h b/tools/objtool/special.h index 44da89afeda2..1dc1bb3e74c6 100644 --- a/tools/objtool/special.h +++ b/tools/objtool/special.h @@ -7,6 +7,7 @@ #define _SPECIAL_H #include +#include "check.h" #include "elf.h" struct special_alt { @@ -30,4 +31,7 @@ int special_get_alts(struct elf *elf, struct list_head *alts); void arch_handle_alternative(unsigned short feature, struct special_alt *alt); +bool arch_support_alt_relocation(struct special_alt *special_alt, + struct instruction *insn, + struct reloc *reloc); #endif /* _SPECIAL_H */ From d871f7b5a6a2a30f4eba577fd56941fa3657e394 Mon Sep 17 00:00:00 2001 From: Raphael Gault Date: Fri, 4 Sep 2020 16:30:24 +0100 Subject: [PATCH 093/265] objtool: Refactor jump table code to support other architectures The way to identify jump tables and retrieve all the data necessary to handle the different execution branches is not the same on all architectures. In order to be able to add other architecture support, define an arch-dependent function to process jump-tables. Reviewed-by: Miroslav Benes Signed-off-by: Raphael Gault [J.T.: Move arm64 bits out of this patch, Have only one function to find the start of the jump table, for now assume that the jump table format will be the same as x86] Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- tools/objtool/arch/x86/special.c | 95 ++++++++++++++++++++++++++++++++ tools/objtool/check.c | 90 ++---------------------------- tools/objtool/check.h | 1 - tools/objtool/special.h | 4 ++ 4 files changed, 103 insertions(+), 87 deletions(-) diff --git a/tools/objtool/arch/x86/special.c b/tools/objtool/arch/x86/special.c index 34e0e162e6fd..fd4af88c0ea5 100644 --- a/tools/objtool/arch/x86/special.c +++ b/tools/objtool/arch/x86/special.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#include + #include "../../special.h" #include "../../builtin.h" @@ -48,3 +50,96 @@ bool arch_support_alt_relocation(struct special_alt *special_alt, return insn->offset == special_alt->new_off && (insn->type == INSN_CALL || is_static_jump(insn)); } + +/* + * There are 3 basic jump table patterns: + * + * 1. jmpq *[rodata addr](,%reg,8) + * + * This is the most common case by far. It jumps to an address in a simple + * jump table which is stored in .rodata. + * + * 2. jmpq *[rodata addr](%rip) + * + * This is caused by a rare GCC quirk, currently only seen in three driver + * functions in the kernel, only with certain obscure non-distro configs. + * + * As part of an optimization, GCC makes a copy of an existing switch jump + * table, modifies it, and then hard-codes the jump (albeit with an indirect + * jump) to use a single entry in the table. The rest of the jump table and + * some of its jump targets remain as dead code. + * + * In such a case we can just crudely ignore all unreachable instruction + * warnings for the entire object file. Ideally we would just ignore them + * for the function, but that would require redesigning the code quite a + * bit. And honestly that's just not worth doing: unreachable instruction + * warnings are of questionable value anyway, and this is such a rare issue. + * + * 3. mov [rodata addr],%reg1 + * ... some instructions ... + * jmpq *(%reg1,%reg2,8) + * + * This is a fairly uncommon pattern which is new for GCC 6. As of this + * writing, there are 11 occurrences of it in the allmodconfig kernel. + * + * As of GCC 7 there are quite a few more of these and the 'in between' code + * is significant. Esp. with KASAN enabled some of the code between the mov + * and jmpq uses .rodata itself, which can confuse things. + * + * TODO: Once we have DWARF CFI and smarter instruction decoding logic, + * ensure the same register is used in the mov and jump instructions. + * + * NOTE: RETPOLINE made it harder still to decode dynamic jumps. + */ +struct reloc *arch_find_switch_table(struct objtool_file *file, + struct instruction *insn) +{ + struct reloc *text_reloc, *rodata_reloc; + struct section *table_sec; + unsigned long table_offset; + + /* look for a relocation which references .rodata */ + text_reloc = find_reloc_by_dest_range(file->elf, insn->sec, + insn->offset, insn->len); + if (!text_reloc || text_reloc->sym->type != STT_SECTION || + !text_reloc->sym->sec->rodata) + return NULL; + + table_offset = text_reloc->addend; + table_sec = text_reloc->sym->sec; + + if (text_reloc->type == R_X86_64_PC32) + table_offset += 4; + + /* + * Make sure the .rodata address isn't associated with a + * symbol. GCC jump tables are anonymous data. + * + * Also support C jump tables which are in the same format as + * switch jump tables. For objtool to recognize them, they + * need to be placed in the C_JUMP_TABLE_SECTION section. They + * have symbols associated with them. + */ + if (find_symbol_containing(table_sec, table_offset) && + strcmp(table_sec->name, C_JUMP_TABLE_SECTION)) + return NULL; + + /* + * Each table entry has a rela associated with it. The rela + * should reference text in the same function as the original + * instruction. + */ + rodata_reloc = find_reloc_by_dest(file->elf, table_sec, table_offset); + if (!rodata_reloc) + return NULL; + + /* + * Use of RIP-relative switch jumps is quite rare, and + * indicates a rare GCC quirk/bug which can leave dead + * code behind. + */ + if (text_reloc->type == R_X86_64_PC32) + file->ignore_unreachables = true; + + return rodata_reloc; +} diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 1796a7c464eb..a94ad88d036c 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -20,8 +20,6 @@ #define FAKE_JUMP_OFFSET -1 -#define C_JUMP_TABLE_SECTION ".rodata..c_jump_table" - struct alternative { struct list_head list; struct instruction *insn; @@ -1190,56 +1188,15 @@ static int add_jump_table(struct objtool_file *file, struct instruction *insn, } /* - * find_jump_table() - Given a dynamic jump, find the switch jump table in - * .rodata associated with it. - * - * There are 3 basic patterns: - * - * 1. jmpq *[rodata addr](,%reg,8) - * - * This is the most common case by far. It jumps to an address in a simple - * jump table which is stored in .rodata. - * - * 2. jmpq *[rodata addr](%rip) - * - * This is caused by a rare GCC quirk, currently only seen in three driver - * functions in the kernel, only with certain obscure non-distro configs. - * - * As part of an optimization, GCC makes a copy of an existing switch jump - * table, modifies it, and then hard-codes the jump (albeit with an indirect - * jump) to use a single entry in the table. The rest of the jump table and - * some of its jump targets remain as dead code. - * - * In such a case we can just crudely ignore all unreachable instruction - * warnings for the entire object file. Ideally we would just ignore them - * for the function, but that would require redesigning the code quite a - * bit. And honestly that's just not worth doing: unreachable instruction - * warnings are of questionable value anyway, and this is such a rare issue. - * - * 3. mov [rodata addr],%reg1 - * ... some instructions ... - * jmpq *(%reg1,%reg2,8) - * - * This is a fairly uncommon pattern which is new for GCC 6. As of this - * writing, there are 11 occurrences of it in the allmodconfig kernel. - * - * As of GCC 7 there are quite a few more of these and the 'in between' code - * is significant. Esp. with KASAN enabled some of the code between the mov - * and jmpq uses .rodata itself, which can confuse things. - * - * TODO: Once we have DWARF CFI and smarter instruction decoding logic, - * ensure the same register is used in the mov and jump instructions. - * - * NOTE: RETPOLINE made it harder still to decode dynamic jumps. + * find_jump_table() - Given a dynamic jump, find the switch jump table + * associated with it. */ static struct reloc *find_jump_table(struct objtool_file *file, struct symbol *func, struct instruction *insn) { - struct reloc *text_reloc, *table_reloc; + struct reloc *table_reloc; struct instruction *dest_insn, *orig_insn = insn; - struct section *table_sec; - unsigned long table_offset; /* * Backward search using the @first_jump_src links, these help avoid @@ -1260,52 +1217,13 @@ static struct reloc *find_jump_table(struct objtool_file *file, insn->jump_dest->offset > orig_insn->offset)) break; - /* look for a relocation which references .rodata */ - text_reloc = find_reloc_by_dest_range(file->elf, insn->sec, - insn->offset, insn->len); - if (!text_reloc || text_reloc->sym->type != STT_SECTION || - !text_reloc->sym->sec->rodata) - continue; - - table_offset = text_reloc->addend; - table_sec = text_reloc->sym->sec; - - if (text_reloc->type == R_X86_64_PC32) - table_offset += 4; - - /* - * Make sure the .rodata address isn't associated with a - * symbol. GCC jump tables are anonymous data. - * - * Also support C jump tables which are in the same format as - * switch jump tables. For objtool to recognize them, they - * need to be placed in the C_JUMP_TABLE_SECTION section. They - * have symbols associated with them. - */ - if (find_symbol_containing(table_sec, table_offset) && - strcmp(table_sec->name, C_JUMP_TABLE_SECTION)) - continue; - - /* - * Each table entry has a reloc associated with it. The reloc - * should reference text in the same function as the original - * instruction. - */ - table_reloc = find_reloc_by_dest(file->elf, table_sec, table_offset); + table_reloc = arch_find_switch_table(file, insn); if (!table_reloc) continue; dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend); if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func) continue; - /* - * Use of RIP-relative switch jumps is quite rare, and - * indicates a rare GCC quirk/bug which can leave dead code - * behind. - */ - if (text_reloc->type == R_X86_64_PC32) - file->ignore_unreachables = true; - return table_reloc; } diff --git a/tools/objtool/check.h b/tools/objtool/check.h index 1de1188b23cd..5ec00a4b891b 100644 --- a/tools/objtool/check.h +++ b/tools/objtool/check.h @@ -66,5 +66,4 @@ struct instruction *find_insn(struct objtool_file *file, insn->sec == sec; \ insn = list_next_entry(insn, list)) - #endif /* _CHECK_H */ diff --git a/tools/objtool/special.h b/tools/objtool/special.h index 1dc1bb3e74c6..abddf38ef334 100644 --- a/tools/objtool/special.h +++ b/tools/objtool/special.h @@ -10,6 +10,8 @@ #include "check.h" #include "elf.h" +#define C_JUMP_TABLE_SECTION ".rodata..c_jump_table" + struct special_alt { struct list_head list; @@ -34,4 +36,6 @@ void arch_handle_alternative(unsigned short feature, struct special_alt *alt); bool arch_support_alt_relocation(struct special_alt *special_alt, struct instruction *insn, struct reloc *reloc); +struct reloc *arch_find_switch_table(struct objtool_file *file, + struct instruction *insn); #endif /* _SPECIAL_H */ From 00089c048eb4a8250325efb32a2724fd0da68cce Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Fri, 4 Sep 2020 16:30:25 +0100 Subject: [PATCH 094/265] objtool: Rename frame.h -> objtool.h Header frame.h is getting more code annotations to help objtool analyze object files. Rename the file to objtool.h. [ jpoimboe: add objtool.h to MAINTAINERS ] Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- MAINTAINERS | 1 + arch/x86/include/asm/nospec-branch.h | 2 +- arch/x86/kernel/kprobes/core.c | 2 +- arch/x86/kernel/kprobes/opt.c | 2 +- arch/x86/kernel/reboot.c | 2 +- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/xen/enlighten_pv.c | 2 +- drivers/gpu/drm/vmwgfx/vmwgfx_msg.c | 3 +-- include/linux/{frame.h => objtool.h} | 6 +++--- kernel/bpf/core.c | 2 +- kernel/kexec_core.c | 2 +- 13 files changed, 15 insertions(+), 15 deletions(-) rename include/linux/{frame.h => objtool.h} (93%) diff --git a/MAINTAINERS b/MAINTAINERS index e4647c84c987..2605a086eb8a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12481,6 +12481,7 @@ M: Josh Poimboeuf M: Peter Zijlstra S: Supported F: tools/objtool/ +F: include/linux/objtool.h OCELOT ETHERNET SWITCH DRIVER M: Microchip Linux Driver Support diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index e7752b4038ff..86651e86289d 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -4,7 +4,7 @@ #define _ASM_X86_NOSPEC_BRANCH_H_ #include -#include +#include #include #include diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index fdadc37d72af..ae2488643029 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -38,9 +38,9 @@ #include #include #include -#include #include #include +#include #include #include diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c index c068e21c2c40..9b1cb8f519b0 100644 --- a/arch/x86/kernel/kprobes/opt.c +++ b/arch/x86/kernel/kprobes/opt.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index a515e2d230b7..db115943e8bd 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 0194336b64a4..17ba613de9a9 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 23b58c28a1c9..ae4ff7c624a4 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -#include +#include #include #include diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 819c185adf09..df29fb49b43d 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -13,7 +13,6 @@ * Yaniv Kamay */ -#include #include #include #include @@ -22,6 +21,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 22e741e0b10c..58382d26f153 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c b/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c index e9f448a5ebb3..15b5bde69324 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c @@ -24,7 +24,7 @@ * */ -#include +#include #include #include #include @@ -599,4 +599,3 @@ out_open: return -EINVAL; } - diff --git a/include/linux/frame.h b/include/linux/objtool.h similarity index 93% rename from include/linux/frame.h rename to include/linux/objtool.h index 303cda600e56..358175c9c2b5 100644 --- a/include/linux/frame.h +++ b/include/linux/objtool.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LINUX_FRAME_H -#define _LINUX_FRAME_H +#ifndef _LINUX_OBJTOOL_H +#define _LINUX_OBJTOOL_H #ifdef CONFIG_STACK_VALIDATION /* @@ -32,4 +32,4 @@ #endif /* CONFIG_STACK_VALIDATION */ -#endif /* _LINUX_FRAME_H */ +#endif /* _LINUX_OBJTOOL_H */ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index ed0b3578867c..03e284873644 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index c19c0dad1ebe..c5e5e5a11535 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include From 5567c6c39f3404e4492c18c0c1abff5556684f6e Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Fri, 4 Sep 2020 16:30:26 +0100 Subject: [PATCH 095/265] objtool: Only include valid definitions depending on source file type Header include/linux/objtool.h contains both C and assembly definition that are visible regardless of the file including them. Place definition under conditional __ASSEMBLY__. Reviewed-by: Miroslav Benes Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- include/linux/objtool.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 358175c9c2b5..15e9997a9fb4 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -3,6 +3,8 @@ #define _LINUX_OBJTOOL_H #ifdef CONFIG_STACK_VALIDATION + +#ifndef __ASSEMBLY__ /* * This macro marks the given function's stack frame as "non-standard", which * tells objtool to ignore the function when doing stack metadata validation. @@ -15,6 +17,8 @@ static void __used __section(.discard.func_stack_frame_non_standard) \ *__func_stack_frame_non_standard_##func = func +#else /* __ASSEMBLY__ */ + /* * This macro indicates that the following intra-function call is valid. * Any non-annotated intra-function call will cause objtool to issue a warning. @@ -25,6 +29,8 @@ .long 999b; \ .popsection; +#endif /* __ASSEMBLY__ */ + #else /* !CONFIG_STACK_VALIDATION */ #define STACK_FRAME_NON_STANDARD(func) From ee819aedf34a8f35cd54ee3967c7beb4d1d4a635 Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Fri, 4 Sep 2020 16:30:27 +0100 Subject: [PATCH 096/265] objtool: Make unwind hint definitions available to other architectures Unwind hints are useful to provide objtool with information about stack states in non-standard functions/code. While the type of information being provided might be very arch specific, the mechanism to provide the information can be useful for other architectures. Move the relevant unwint hint definitions for all architectures to see. [ jpoimboe: REGS_IRET -> REGS_PARTIAL ] Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- arch/x86/include/asm/orc_types.h | 34 ------- arch/x86/include/asm/unwind_hints.h | 56 +++-------- arch/x86/kernel/unwind_orc.c | 11 ++- include/linux/objtool.h | 88 +++++++++++++++++ tools/arch/x86/include/asm/orc_types.h | 34 ------- tools/include/linux/objtool.h | 129 +++++++++++++++++++++++++ tools/objtool/check.c | 4 +- tools/objtool/orc_dump.c | 9 +- tools/objtool/orc_gen.c | 5 +- tools/objtool/sync-check.sh | 4 +- 10 files changed, 249 insertions(+), 125 deletions(-) create mode 100644 tools/include/linux/objtool.h diff --git a/arch/x86/include/asm/orc_types.h b/arch/x86/include/asm/orc_types.h index d25534940bde..fdbffec4cfde 100644 --- a/arch/x86/include/asm/orc_types.h +++ b/arch/x86/include/asm/orc_types.h @@ -39,27 +39,6 @@ #define ORC_REG_SP_INDIRECT 9 #define ORC_REG_MAX 15 -/* - * ORC_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP (the - * caller's SP right before it made the call). Used for all callable - * functions, i.e. all C code and all callable asm functions. - * - * ORC_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset points - * to a fully populated pt_regs from a syscall, interrupt, or exception. - * - * ORC_TYPE_REGS_IRET: Used in entry code to indicate that sp_reg+sp_offset - * points to the iret return frame. - * - * The UNWIND_HINT macros are used only for the unwind_hint struct. They - * aren't used in struct orc_entry due to size and complexity constraints. - * Objtool converts them to real types when it converts the hints to orc - * entries. - */ -#define ORC_TYPE_CALL 0 -#define ORC_TYPE_REGS 1 -#define ORC_TYPE_REGS_IRET 2 -#define UNWIND_HINT_TYPE_RET_OFFSET 3 - #ifndef __ASSEMBLY__ /* * This struct is more or less a vastly simplified version of the DWARF Call @@ -78,19 +57,6 @@ struct orc_entry { unsigned end:1; } __packed; -/* - * This struct is used by asm and inline asm code to manually annotate the - * location of registers on the stack for the ORC unwinder. - * - * Type can be either ORC_TYPE_* or UNWIND_HINT_TYPE_*. - */ -struct unwind_hint { - u32 ip; - s16 sp_offset; - u8 sp_reg; - u8 type; - u8 end; -}; #endif /* __ASSEMBLY__ */ #endif /* _ORC_TYPES_H */ diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h index 7d903fdb3f43..664d4610d700 100644 --- a/arch/x86/include/asm/unwind_hints.h +++ b/arch/x86/include/asm/unwind_hints.h @@ -1,51 +1,17 @@ #ifndef _ASM_X86_UNWIND_HINTS_H #define _ASM_X86_UNWIND_HINTS_H +#include + #include "orc_types.h" #ifdef __ASSEMBLY__ -/* - * In asm, there are two kinds of code: normal C-type callable functions and - * the rest. The normal callable functions can be called by other code, and - * don't do anything unusual with the stack. Such normal callable functions - * are annotated with the ENTRY/ENDPROC macros. Most asm code falls in this - * category. In this case, no special debugging annotations are needed because - * objtool can automatically generate the ORC data for the ORC unwinder to read - * at runtime. - * - * Anything which doesn't fall into the above category, such as syscall and - * interrupt handlers, tends to not be called directly by other functions, and - * often does unusual non-C-function-type things with the stack pointer. Such - * code needs to be annotated such that objtool can understand it. The - * following CFI hint macros are for this type of code. - * - * These macros provide hints to objtool about the state of the stack at each - * instruction. Objtool starts from the hints and follows the code flow, - * making automatic CFI adjustments when it sees pushes and pops, filling out - * the debuginfo as necessary. It will also warn if it sees any - * inconsistencies. - */ -.macro UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=0 type=ORC_TYPE_CALL end=0 -#ifdef CONFIG_STACK_VALIDATION -.Lunwind_hint_ip_\@: - .pushsection .discard.unwind_hints - /* struct unwind_hint */ - .long .Lunwind_hint_ip_\@ - . - .short \sp_offset - .byte \sp_reg - .byte \type - .byte \end - .balign 4 - .popsection -#endif -.endm - .macro UNWIND_HINT_EMPTY - UNWIND_HINT sp_reg=ORC_REG_UNDEFINED end=1 + UNWIND_HINT sp_reg=ORC_REG_UNDEFINED type=UNWIND_HINT_TYPE_CALL end=1 .endm -.macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 iret=0 +.macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 partial=0 .if \base == %rsp .if \indirect .set sp_reg, ORC_REG_SP_INDIRECT @@ -66,24 +32,24 @@ .set sp_offset, \offset - .if \iret - .set type, ORC_TYPE_REGS_IRET + .if \partial + .set type, UNWIND_HINT_TYPE_REGS_PARTIAL .elseif \extra == 0 - .set type, ORC_TYPE_REGS_IRET + .set type, UNWIND_HINT_TYPE_REGS_PARTIAL .set sp_offset, \offset + (16*8) .else - .set type, ORC_TYPE_REGS + .set type, UNWIND_HINT_TYPE_REGS .endif UNWIND_HINT sp_reg=sp_reg sp_offset=sp_offset type=type .endm .macro UNWIND_HINT_IRET_REGS base=%rsp offset=0 - UNWIND_HINT_REGS base=\base offset=\offset iret=1 + UNWIND_HINT_REGS base=\base offset=\offset partial=1 .endm .macro UNWIND_HINT_FUNC sp_offset=8 - UNWIND_HINT sp_offset=\sp_offset + UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=\sp_offset type=UNWIND_HINT_TYPE_CALL .endm /* @@ -92,7 +58,7 @@ * initial_func_cfi. */ .macro UNWIND_HINT_RET_OFFSET sp_offset=8 - UNWIND_HINT type=UNWIND_HINT_TYPE_RET_OFFSET sp_offset=\sp_offset + UNWIND_HINT sp_reg=ORC_REG_SP type=UNWIND_HINT_TYPE_RET_OFFSET sp_offset=\sp_offset .endm #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c index ec88bbe08a32..6a339ce328e0 100644 --- a/arch/x86/kernel/unwind_orc.c +++ b/arch/x86/kernel/unwind_orc.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only +#include #include #include #include @@ -127,12 +128,12 @@ static struct orc_entry null_orc_entry = { .sp_offset = sizeof(long), .sp_reg = ORC_REG_SP, .bp_reg = ORC_REG_UNDEFINED, - .type = ORC_TYPE_CALL + .type = UNWIND_HINT_TYPE_CALL }; /* Fake frame pointer entry -- used as a fallback for generated code */ static struct orc_entry orc_fp_entry = { - .type = ORC_TYPE_CALL, + .type = UNWIND_HINT_TYPE_CALL, .sp_reg = ORC_REG_BP, .sp_offset = 16, .bp_reg = ORC_REG_PREV_SP, @@ -531,7 +532,7 @@ bool unwind_next_frame(struct unwind_state *state) /* Find IP, SP and possibly regs: */ switch (orc->type) { - case ORC_TYPE_CALL: + case UNWIND_HINT_TYPE_CALL: ip_p = sp - sizeof(long); if (!deref_stack_reg(state, ip_p, &state->ip)) @@ -546,7 +547,7 @@ bool unwind_next_frame(struct unwind_state *state) state->signal = false; break; - case ORC_TYPE_REGS: + case UNWIND_HINT_TYPE_REGS: if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) { orc_warn_current("can't access registers at %pB\n", (void *)orig_ip); @@ -559,7 +560,7 @@ bool unwind_next_frame(struct unwind_state *state) state->signal = true; break; - case ORC_TYPE_REGS_IRET: + case UNWIND_HINT_TYPE_REGS_PARTIAL: if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) { orc_warn_current("can't access iret registers at %pB\n", (void *)orig_ip); diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 15e9997a9fb4..ab82c793c897 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -2,9 +2,55 @@ #ifndef _LINUX_OBJTOOL_H #define _LINUX_OBJTOOL_H +#ifndef __ASSEMBLY__ + +#include + +/* + * This struct is used by asm and inline asm code to manually annotate the + * location of registers on the stack. + */ +struct unwind_hint { + u32 ip; + s16 sp_offset; + u8 sp_reg; + u8 type; + u8 end; +}; +#endif + +/* + * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP + * (the caller's SP right before it made the call). Used for all callable + * functions, i.e. all C code and all callable asm functions. + * + * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset + * points to a fully populated pt_regs from a syscall, interrupt, or exception. + * + * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that + * sp_reg+sp_offset points to the iret return frame. + */ +#define UNWIND_HINT_TYPE_CALL 0 +#define UNWIND_HINT_TYPE_REGS 1 +#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +#define UNWIND_HINT_TYPE_RET_OFFSET 3 + #ifdef CONFIG_STACK_VALIDATION #ifndef __ASSEMBLY__ + +#define UNWIND_HINT(sp_reg, sp_offset, type, end) \ + "987: \n\t" \ + ".pushsection .discard.unwind_hints\n\t" \ + /* struct unwind_hint */ \ + ".long 987b - .\n\t" \ + ".short " __stringify(sp_offset) "\n\t" \ + ".byte " __stringify(sp_reg) "\n\t" \ + ".byte " __stringify(type) "\n\t" \ + ".byte " __stringify(end) "\n\t" \ + ".balign 4 \n\t" \ + ".popsection\n\t" + /* * This macro marks the given function's stack frame as "non-standard", which * tells objtool to ignore the function when doing stack metadata validation. @@ -29,12 +75,54 @@ .long 999b; \ .popsection; +/* + * In asm, there are two kinds of code: normal C-type callable functions and + * the rest. The normal callable functions can be called by other code, and + * don't do anything unusual with the stack. Such normal callable functions + * are annotated with the ENTRY/ENDPROC macros. Most asm code falls in this + * category. In this case, no special debugging annotations are needed because + * objtool can automatically generate the ORC data for the ORC unwinder to read + * at runtime. + * + * Anything which doesn't fall into the above category, such as syscall and + * interrupt handlers, tends to not be called directly by other functions, and + * often does unusual non-C-function-type things with the stack pointer. Such + * code needs to be annotated such that objtool can understand it. The + * following CFI hint macros are for this type of code. + * + * These macros provide hints to objtool about the state of the stack at each + * instruction. Objtool starts from the hints and follows the code flow, + * making automatic CFI adjustments when it sees pushes and pops, filling out + * the debuginfo as necessary. It will also warn if it sees any + * inconsistencies. + */ +.macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0 +.Lunwind_hint_ip_\@: + .pushsection .discard.unwind_hints + /* struct unwind_hint */ + .long .Lunwind_hint_ip_\@ - . + .short \sp_offset + .byte \sp_reg + .byte \type + .byte \end + .balign 4 + .popsection +.endm + #endif /* __ASSEMBLY__ */ #else /* !CONFIG_STACK_VALIDATION */ +#ifndef __ASSEMBLY__ + +#define UNWIND_HINT(sp_reg, sp_offset, type, end) \ + "\n\t" #define STACK_FRAME_NON_STANDARD(func) +#else #define ANNOTATE_INTRA_FUNCTION_CALL +.macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0 +.endm +#endif #endif /* CONFIG_STACK_VALIDATION */ diff --git a/tools/arch/x86/include/asm/orc_types.h b/tools/arch/x86/include/asm/orc_types.h index d25534940bde..fdbffec4cfde 100644 --- a/tools/arch/x86/include/asm/orc_types.h +++ b/tools/arch/x86/include/asm/orc_types.h @@ -39,27 +39,6 @@ #define ORC_REG_SP_INDIRECT 9 #define ORC_REG_MAX 15 -/* - * ORC_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP (the - * caller's SP right before it made the call). Used for all callable - * functions, i.e. all C code and all callable asm functions. - * - * ORC_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset points - * to a fully populated pt_regs from a syscall, interrupt, or exception. - * - * ORC_TYPE_REGS_IRET: Used in entry code to indicate that sp_reg+sp_offset - * points to the iret return frame. - * - * The UNWIND_HINT macros are used only for the unwind_hint struct. They - * aren't used in struct orc_entry due to size and complexity constraints. - * Objtool converts them to real types when it converts the hints to orc - * entries. - */ -#define ORC_TYPE_CALL 0 -#define ORC_TYPE_REGS 1 -#define ORC_TYPE_REGS_IRET 2 -#define UNWIND_HINT_TYPE_RET_OFFSET 3 - #ifndef __ASSEMBLY__ /* * This struct is more or less a vastly simplified version of the DWARF Call @@ -78,19 +57,6 @@ struct orc_entry { unsigned end:1; } __packed; -/* - * This struct is used by asm and inline asm code to manually annotate the - * location of registers on the stack for the ORC unwinder. - * - * Type can be either ORC_TYPE_* or UNWIND_HINT_TYPE_*. - */ -struct unwind_hint { - u32 ip; - s16 sp_offset; - u8 sp_reg; - u8 type; - u8 end; -}; #endif /* __ASSEMBLY__ */ #endif /* _ORC_TYPES_H */ diff --git a/tools/include/linux/objtool.h b/tools/include/linux/objtool.h new file mode 100644 index 000000000000..ab82c793c897 --- /dev/null +++ b/tools/include/linux/objtool.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_OBJTOOL_H +#define _LINUX_OBJTOOL_H + +#ifndef __ASSEMBLY__ + +#include + +/* + * This struct is used by asm and inline asm code to manually annotate the + * location of registers on the stack. + */ +struct unwind_hint { + u32 ip; + s16 sp_offset; + u8 sp_reg; + u8 type; + u8 end; +}; +#endif + +/* + * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP + * (the caller's SP right before it made the call). Used for all callable + * functions, i.e. all C code and all callable asm functions. + * + * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset + * points to a fully populated pt_regs from a syscall, interrupt, or exception. + * + * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that + * sp_reg+sp_offset points to the iret return frame. + */ +#define UNWIND_HINT_TYPE_CALL 0 +#define UNWIND_HINT_TYPE_REGS 1 +#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +#define UNWIND_HINT_TYPE_RET_OFFSET 3 + +#ifdef CONFIG_STACK_VALIDATION + +#ifndef __ASSEMBLY__ + +#define UNWIND_HINT(sp_reg, sp_offset, type, end) \ + "987: \n\t" \ + ".pushsection .discard.unwind_hints\n\t" \ + /* struct unwind_hint */ \ + ".long 987b - .\n\t" \ + ".short " __stringify(sp_offset) "\n\t" \ + ".byte " __stringify(sp_reg) "\n\t" \ + ".byte " __stringify(type) "\n\t" \ + ".byte " __stringify(end) "\n\t" \ + ".balign 4 \n\t" \ + ".popsection\n\t" + +/* + * This macro marks the given function's stack frame as "non-standard", which + * tells objtool to ignore the function when doing stack metadata validation. + * It should only be used in special cases where you're 100% sure it won't + * affect the reliability of frame pointers and kernel stack traces. + * + * For more information, see tools/objtool/Documentation/stack-validation.txt. + */ +#define STACK_FRAME_NON_STANDARD(func) \ + static void __used __section(.discard.func_stack_frame_non_standard) \ + *__func_stack_frame_non_standard_##func = func + +#else /* __ASSEMBLY__ */ + +/* + * This macro indicates that the following intra-function call is valid. + * Any non-annotated intra-function call will cause objtool to issue a warning. + */ +#define ANNOTATE_INTRA_FUNCTION_CALL \ + 999: \ + .pushsection .discard.intra_function_calls; \ + .long 999b; \ + .popsection; + +/* + * In asm, there are two kinds of code: normal C-type callable functions and + * the rest. The normal callable functions can be called by other code, and + * don't do anything unusual with the stack. Such normal callable functions + * are annotated with the ENTRY/ENDPROC macros. Most asm code falls in this + * category. In this case, no special debugging annotations are needed because + * objtool can automatically generate the ORC data for the ORC unwinder to read + * at runtime. + * + * Anything which doesn't fall into the above category, such as syscall and + * interrupt handlers, tends to not be called directly by other functions, and + * often does unusual non-C-function-type things with the stack pointer. Such + * code needs to be annotated such that objtool can understand it. The + * following CFI hint macros are for this type of code. + * + * These macros provide hints to objtool about the state of the stack at each + * instruction. Objtool starts from the hints and follows the code flow, + * making automatic CFI adjustments when it sees pushes and pops, filling out + * the debuginfo as necessary. It will also warn if it sees any + * inconsistencies. + */ +.macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0 +.Lunwind_hint_ip_\@: + .pushsection .discard.unwind_hints + /* struct unwind_hint */ + .long .Lunwind_hint_ip_\@ - . + .short \sp_offset + .byte \sp_reg + .byte \type + .byte \end + .balign 4 + .popsection +.endm + +#endif /* __ASSEMBLY__ */ + +#else /* !CONFIG_STACK_VALIDATION */ + +#ifndef __ASSEMBLY__ + +#define UNWIND_HINT(sp_reg, sp_offset, type, end) \ + "\n\t" +#define STACK_FRAME_NON_STANDARD(func) +#else +#define ANNOTATE_INTRA_FUNCTION_CALL +.macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0 +.endm +#endif + +#endif /* CONFIG_STACK_VALIDATION */ + +#endif /* _LINUX_OBJTOOL_H */ diff --git a/tools/objtool/check.c b/tools/objtool/check.c index a94ad88d036c..95c6e0d31c0a 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -14,6 +14,7 @@ #include "warn.h" #include "arch_elf.h" +#include #include #include #include @@ -1805,7 +1806,8 @@ static int update_cfi_state(struct instruction *insn, struct cfi_state *cfi, return 0; } - if (cfi->type == ORC_TYPE_REGS || cfi->type == ORC_TYPE_REGS_IRET) + if (cfi->type == UNWIND_HINT_TYPE_REGS || + cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL) return update_cfi_state_regs(insn, cfi, op); switch (op->dest.type) { diff --git a/tools/objtool/orc_dump.c b/tools/objtool/orc_dump.c index fca46e006fc2..5e6a95368d35 100644 --- a/tools/objtool/orc_dump.c +++ b/tools/objtool/orc_dump.c @@ -4,6 +4,7 @@ */ #include +#include #include #include "objtool.h" #include "warn.h" @@ -37,12 +38,12 @@ static const char *reg_name(unsigned int reg) static const char *orc_type_name(unsigned int type) { switch (type) { - case ORC_TYPE_CALL: + case UNWIND_HINT_TYPE_CALL: return "call"; - case ORC_TYPE_REGS: + case UNWIND_HINT_TYPE_REGS: return "regs"; - case ORC_TYPE_REGS_IRET: - return "iret"; + case UNWIND_HINT_TYPE_REGS_PARTIAL: + return "regs (partial)"; default: return "?"; } diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c index 22fe4398197f..235663b96adc 100644 --- a/tools/objtool/orc_gen.c +++ b/tools/objtool/orc_gen.c @@ -6,6 +6,9 @@ #include #include +#include +#include + #include "check.h" #include "warn.h" @@ -146,7 +149,7 @@ int create_orc_sections(struct objtool_file *file) struct orc_entry empty = { .sp_reg = ORC_REG_UNDEFINED, .bp_reg = ORC_REG_UNDEFINED, - .type = ORC_TYPE_CALL, + .type = UNWIND_HINT_TYPE_CALL, }; sec = find_section_by_name(file->elf, ".orc_unwind"); diff --git a/tools/objtool/sync-check.sh b/tools/objtool/sync-check.sh index cea1c12607b9..606a4b5e929f 100755 --- a/tools/objtool/sync-check.sh +++ b/tools/objtool/sync-check.sh @@ -6,8 +6,10 @@ if [ -z "$SRCARCH" ]; then exit 1 fi +FILES="include/linux/objtool.h" + if [ "$SRCARCH" = "x86" ]; then -FILES=" +FILES="$FILES arch/x86/include/asm/inat_types.h arch/x86/include/asm/orc_types.h arch/x86/include/asm/emulate_prefix.h From edea9e6bcbeaa41718b022a8b99ffddef2330bbc Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Fri, 4 Sep 2020 16:30:28 +0100 Subject: [PATCH 097/265] objtool: Decode unwind hint register depending on architecture The set of registers that can be included in an unwind hint and their encoding will depend on the architecture. Have arch specific code to decode that register. Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- tools/objtool/arch.h | 2 ++ tools/objtool/arch/x86/decode.c | 37 +++++++++++++++++++++++++++++++++ tools/objtool/check.c | 27 +----------------------- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/tools/objtool/arch.h b/tools/objtool/arch.h index b18c5f61d42d..4a84c3081b8e 100644 --- a/tools/objtool/arch.h +++ b/tools/objtool/arch.h @@ -88,4 +88,6 @@ unsigned long arch_dest_reloc_offset(int addend); const char *arch_nop_insn(int len); +int arch_decode_hint_reg(struct instruction *insn, u8 sp_reg); + #endif /* _ARCH_H */ diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c index 1967370440b3..cde9c36e40ae 100644 --- a/tools/objtool/arch/x86/decode.c +++ b/tools/objtool/arch/x86/decode.c @@ -15,6 +15,7 @@ #include "../../elf.h" #include "../../arch.h" #include "../../warn.h" +#include static unsigned char op_to_cfi_reg[][2] = { {CFI_AX, CFI_R8}, @@ -583,3 +584,39 @@ const char *arch_nop_insn(int len) return nops[len-1]; } + +int arch_decode_hint_reg(struct instruction *insn, u8 sp_reg) +{ + struct cfi_reg *cfa = &insn->cfi.cfa; + + switch (sp_reg) { + case ORC_REG_UNDEFINED: + cfa->base = CFI_UNDEFINED; + break; + case ORC_REG_SP: + cfa->base = CFI_SP; + break; + case ORC_REG_BP: + cfa->base = CFI_BP; + break; + case ORC_REG_SP_INDIRECT: + cfa->base = CFI_SP_INDIRECT; + break; + case ORC_REG_R10: + cfa->base = CFI_R10; + break; + case ORC_REG_R13: + cfa->base = CFI_R13; + break; + case ORC_REG_DI: + cfa->base = CFI_DI; + break; + case ORC_REG_DX: + cfa->base = CFI_DX; + break; + default: + return -1; + } + + return 0; +} diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 95c6e0d31c0a..4e2f703b6a25 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -1367,32 +1367,7 @@ static int read_unwind_hints(struct objtool_file *file) insn->hint = true; - switch (hint->sp_reg) { - case ORC_REG_UNDEFINED: - cfa->base = CFI_UNDEFINED; - break; - case ORC_REG_SP: - cfa->base = CFI_SP; - break; - case ORC_REG_BP: - cfa->base = CFI_BP; - break; - case ORC_REG_SP_INDIRECT: - cfa->base = CFI_SP_INDIRECT; - break; - case ORC_REG_R10: - cfa->base = CFI_R10; - break; - case ORC_REG_R13: - cfa->base = CFI_R13; - break; - case ORC_REG_DI: - cfa->base = CFI_DI; - break; - case ORC_REG_DX: - cfa->base = CFI_DX; - break; - default: + if (arch_decode_hint_reg(insn, hint->sp_reg)) { WARN_FUNC("unsupported unwind_hint sp base reg %d", insn->sec, insn->offset, hint->sp_reg); return -1; From 39336f4ffb2478ad384075cf4ba7ef2e5db2bbd7 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Mon, 7 Sep 2020 15:16:12 +0200 Subject: [PATCH 098/265] x86/efi: Add GHCB mappings when SEV-ES is active Calling down to EFI runtime services can result in the firmware performing VMGEXIT calls. The firmware is likely to use the GHCB of the OS (e.g., for setting EFI variables), so each GHCB in the system needs to be identity-mapped in the EFI page tables, as unencrypted, to avoid page faults. Signed-off-by: Tom Lendacky [ jroedel@suse.de: Moved GHCB mapping loop to sev-es.c ] Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Acked-by: Ard Biesheuvel Link: https://lkml.kernel.org/r/20200907131613.12703-72-joro@8bytes.org --- arch/x86/boot/compressed/sev-es.c | 1 + arch/x86/include/asm/sev-es.h | 2 ++ arch/x86/kernel/sev-es.c | 30 ++++++++++++++++++++++++++++++ arch/x86/platform/efi/efi_64.c | 10 ++++++++++ 4 files changed, 43 insertions(+) diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index 5f15e5864e0c..2a6c7c375244 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -12,6 +12,7 @@ */ #include "misc.h" +#include #include #include #include diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h index e919f09ae33c..cf1d957c7091 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -102,11 +102,13 @@ static __always_inline void sev_es_nmi_complete(void) if (static_branch_unlikely(&sev_es_enable_key)) __sev_es_nmi_complete(); } +extern int __init sev_es_efi_map_ghcbs(pgd_t *pgd); #else static inline void sev_es_ist_enter(struct pt_regs *regs) { } static inline void sev_es_ist_exit(void) { } static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) { return 0; } static inline void sev_es_nmi_complete(void) { } +static inline int sev_es_efi_map_ghcbs(pgd_t *pgd) { return 0; } #endif #endif diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index b6518e96dedb..8cac9f80bfc3 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -491,6 +491,36 @@ int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) return 0; } +/* + * This is needed by the OVMF UEFI firmware which will use whatever it finds in + * the GHCB MSR as its GHCB to talk to the hypervisor. So make sure the per-cpu + * runtime GHCBs used by the kernel are also mapped in the EFI page-table. + */ +int __init sev_es_efi_map_ghcbs(pgd_t *pgd) +{ + struct sev_es_runtime_data *data; + unsigned long address, pflags; + int cpu; + u64 pfn; + + if (!sev_es_active()) + return 0; + + pflags = _PAGE_NX | _PAGE_RW; + + for_each_possible_cpu(cpu) { + data = per_cpu(runtime_data, cpu); + + address = __pa(&data->ghcb_page); + pfn = address >> PAGE_SHIFT; + + if (kernel_map_pages_in_pgd(pgd, pfn, address, 1, pflags)) + return 1; + } + + return 0; +} + static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index 6af4da1149ba..8f5759df7776 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -47,6 +47,7 @@ #include #include #include +#include /* * We allocate runtime services regions top-down, starting from -4G, i.e. @@ -229,6 +230,15 @@ int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages) return 1; } + /* + * When SEV-ES is active, the GHCB as set by the kernel will be used + * by firmware. Create a 1:1 unencrypted mapping for each GHCB. + */ + if (sev_es_efi_map_ghcbs(pgd)) { + pr_err("Failed to create 1:1 mapping for the GHCBs!\n"); + return 1; + } + /* * When making calls to the firmware everything needs to be 1:1 * mapped and addressable with 32-bit pointers. Map the kernel From f5ed777586e08e09c4b6f1e87161a145ee1431cf Mon Sep 17 00:00:00 2001 From: Martin Radev Date: Mon, 7 Sep 2020 15:16:13 +0200 Subject: [PATCH 099/265] x86/sev-es: Check required CPU features for SEV-ES Make sure the machine supports RDRAND, otherwise there is no trusted source of randomness in the system. To also check this in the pre-decompression stage, make has_cpuflag() not depend on CONFIG_RANDOMIZE_BASE anymore. Signed-off-by: Martin Radev Signed-off-by: Joerg Roedel Signed-off-by: Borislav Petkov Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/20200907131613.12703-73-joro@8bytes.org --- arch/x86/boot/compressed/cpuflags.c | 4 ---- arch/x86/boot/compressed/misc.h | 5 +++-- arch/x86/boot/compressed/sev-es.c | 3 +++ arch/x86/kernel/sev-es-shared.c | 15 +++++++++++++++ arch/x86/kernel/sev-es.c | 3 +++ 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/arch/x86/boot/compressed/cpuflags.c b/arch/x86/boot/compressed/cpuflags.c index 6448a8196d32..0cc1323896d1 100644 --- a/arch/x86/boot/compressed/cpuflags.c +++ b/arch/x86/boot/compressed/cpuflags.c @@ -1,6 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 -#ifdef CONFIG_RANDOMIZE_BASE - #include "../cpuflags.c" bool has_cpuflag(int flag) @@ -9,5 +7,3 @@ bool has_cpuflag(int flag) return test_bit(flag, cpu.flags); } - -#endif diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index c0e0ffeee50a..6d31f1b4c4d1 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -85,8 +85,6 @@ void choose_random_location(unsigned long input, unsigned long *output, unsigned long output_size, unsigned long *virt_addr); -/* cpuflags.c */ -bool has_cpuflag(int flag); #else static inline void choose_random_location(unsigned long input, unsigned long input_size, @@ -97,6 +95,9 @@ static inline void choose_random_location(unsigned long input, } #endif +/* cpuflags.c */ +bool has_cpuflag(int flag); + #ifdef CONFIG_X86_64 extern int set_page_decrypted(unsigned long address); extern int set_page_encrypted(unsigned long address); diff --git a/arch/x86/boot/compressed/sev-es.c b/arch/x86/boot/compressed/sev-es.c index 2a6c7c375244..954cb2702e23 100644 --- a/arch/x86/boot/compressed/sev-es.c +++ b/arch/x86/boot/compressed/sev-es.c @@ -145,6 +145,9 @@ void sev_es_shutdown_ghcb(void) if (!boot_ghcb) return; + if (!sev_es_check_cpu_features()) + error("SEV-ES CPU Features missing."); + /* * GHCB Page must be flushed from the cache and mapped encrypted again. * Otherwise the running kernel will see strange cache effects when diff --git a/arch/x86/kernel/sev-es-shared.c b/arch/x86/kernel/sev-es-shared.c index 4be8af2f9c57..5f83ccaab877 100644 --- a/arch/x86/kernel/sev-es-shared.c +++ b/arch/x86/kernel/sev-es-shared.c @@ -9,6 +9,21 @@ * and is included directly into both code-bases. */ +#ifndef __BOOT_COMPRESSED +#define error(v) pr_err(v) +#define has_cpuflag(f) boot_cpu_has(f) +#endif + +static bool __init sev_es_check_cpu_features(void) +{ + if (!has_cpuflag(X86_FEATURE_RDRAND)) { + error("RDRAND instruction not supported - no trusted source of randomness available\n"); + return false; + } + + return true; +} + static void sev_es_terminate(unsigned int reason) { u64 val = GHCB_SEV_TERMINATE; diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 8cac9f80bfc3..6fcfdd32769f 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -665,6 +665,9 @@ void __init sev_es_init_vc_handling(void) if (!sev_es_active()) return; + if (!sev_es_check_cpu_features()) + panic("SEV-ES CPU Features missing"); + /* Enable SEV-ES special handling */ static_branch_enable(&sev_es_enable_key); From 7a8379eb41a47d37e93d34f09ca1c3b7d10de073 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 11 Sep 2020 16:44:45 +0200 Subject: [PATCH 100/265] ACPICA: Add support for using logical addresses of GPE blocks The logical address of every GPE block in system memory must be known before passing it to acpi_ev_initialize_gpe_block(), because memory cannot be mapped on the fly from an interrupt handler. Accordingly, the host OS must map every GPE block in system memory upfront and it can store the logical addresses of GPE blocks for future use. If these logical addresses were known to ACPICA, it could use them instead of the corresponding physical addresses of GPE block for GPE register accesses and the memory mapping lookups carried out by acpi_os_read_memory() and acpi_os_write_memory() on every attempt to access a GPE register would not be necessary any more. To allow that to happen, introduce the ACPI_GPE_USE_LOGICAL_ADDRESSES symbol to indicate whether or not the host OS wants ACPICA to use the logical addresses of GPE registers in system memory directly (which is the case if this symbol is defined). Moreover, conditional on whether ACPI_GPE_USE_LOGICAL_ADDRESSES is defined, introduce two new global variables for storing the logical addresses of the FADT GPE blocks 0 and 1, respectively, acpi_gbl_xgpe0_block_logical_address and acpi_gbl_xgpe1_block_logical_address, make acpi_ev_gpe_initialize() pass their values instead of the physical addresses of the GPE blocks in question to acpi_ev_create_gpe_block() and modify acpi_hw_gpe_read() and acpi_hw_gpe_write() to access memory directly via the addresses stored in the struct acpi_gpe_address objects, which are expected to be the logical addresses of GPE registers if ACPI_GPE_USE_LOGICAL_ADDRESSES is defined. With the above changes in place, a host OS wanting ACPICA to access GPE registers directly through their logical addresses needs to define the ACPI_GPE_USE_LOGICAL_ADDRESSES symbol and make sure that the logical addresses of the FADT GPE blocks 0 and 1 are stored in acpi_gbl_xgpe0_block_logical_address and acpi_gbl_xgpe1_block_logical_address, respectively, prior to calling acpi_ev_gpe_initialize(). [If such a host OS also uses acpi_install_gpe_block() to add non-FADT GPE register blocks located in system memory, it must pass their logical addresses instead of their physical addresses to this function.] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpica/acglobal.h | 6 ++++++ drivers/acpi/acpica/evgpeinit.c | 23 +++++++++++++++++------ drivers/acpi/acpica/hwgpe.c | 10 ++++++++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/drivers/acpi/acpica/acglobal.h b/drivers/acpi/acpica/acglobal.h index 1030a0ce1599..2fee91f57b21 100644 --- a/drivers/acpi/acpica/acglobal.h +++ b/drivers/acpi/acpica/acglobal.h @@ -42,6 +42,12 @@ ACPI_GLOBAL(struct acpi_generic_address, acpi_gbl_xpm1a_enable); ACPI_GLOBAL(struct acpi_generic_address, acpi_gbl_xpm1b_status); ACPI_GLOBAL(struct acpi_generic_address, acpi_gbl_xpm1b_enable); +#ifdef ACPI_GPE_USE_LOGICAL_ADDRESSES +ACPI_GLOBAL(unsigned long, acpi_gbl_xgpe0_block_logical_address); +ACPI_GLOBAL(unsigned long, acpi_gbl_xgpe1_block_logical_address); + +#endif /* ACPI_GPE_USE_LOGICAL_ADDRESSES */ + /* * Handle both ACPI 1.0 and ACPI 2.0+ Integer widths. The integer width is * determined by the revision of the DSDT: If the DSDT revision is less than diff --git a/drivers/acpi/acpica/evgpeinit.c b/drivers/acpi/acpica/evgpeinit.c index 6effd8076dcc..6d82d30d8f7b 100644 --- a/drivers/acpi/acpica/evgpeinit.c +++ b/drivers/acpi/acpica/evgpeinit.c @@ -32,6 +32,16 @@ ACPI_MODULE_NAME("evgpeinit") * kernel boot time as well. */ +#ifdef ACPI_GPE_USE_LOGICAL_ADDRESSES +#define ACPI_FADT_GPE_BLOCK_ADDRESS(N) \ + acpi_gbl_FADT.xgpe##N##_block.space_id == \ + ACPI_ADR_SPACE_SYSTEM_MEMORY ? \ + (u64)acpi_gbl_xgpe##N##_block_logical_address : \ + acpi_gbl_FADT.xgpe##N##_block.address +#else +#define ACPI_FADT_GPE_BLOCK_ADDRESS(N) acpi_gbl_FADT.xgpe##N##_block.address +#endif /* ACPI_GPE_USE_LOGICAL_ADDRESSES */ + /******************************************************************************* * * FUNCTION: acpi_ev_gpe_initialize @@ -49,6 +59,7 @@ acpi_status acpi_ev_gpe_initialize(void) u32 register_count1 = 0; u32 gpe_number_max = 0; acpi_status status; + u64 address; ACPI_FUNCTION_TRACE(ev_gpe_initialize); @@ -85,8 +96,9 @@ acpi_status acpi_ev_gpe_initialize(void) * If EITHER the register length OR the block address are zero, then that * particular block is not supported. */ - if (acpi_gbl_FADT.gpe0_block_length && - acpi_gbl_FADT.xgpe0_block.address) { + address = ACPI_FADT_GPE_BLOCK_ADDRESS(0); + + if (acpi_gbl_FADT.gpe0_block_length && address) { /* GPE block 0 exists (has both length and address > 0) */ @@ -97,7 +109,6 @@ acpi_status acpi_ev_gpe_initialize(void) /* Install GPE Block 0 */ status = acpi_ev_create_gpe_block(acpi_gbl_fadt_gpe_device, - acpi_gbl_FADT.xgpe0_block. address, acpi_gbl_FADT.xgpe0_block. space_id, register_count0, 0, @@ -110,8 +121,9 @@ acpi_status acpi_ev_gpe_initialize(void) } } - if (acpi_gbl_FADT.gpe1_block_length && - acpi_gbl_FADT.xgpe1_block.address) { + address = ACPI_FADT_GPE_BLOCK_ADDRESS(1); + + if (acpi_gbl_FADT.gpe1_block_length && address) { /* GPE block 1 exists (has both length and address > 0) */ @@ -137,7 +149,6 @@ acpi_status acpi_ev_gpe_initialize(void) status = acpi_ev_create_gpe_block(acpi_gbl_fadt_gpe_device, - acpi_gbl_FADT.xgpe1_block. address, acpi_gbl_FADT.xgpe1_block. space_id, register_count1, diff --git a/drivers/acpi/acpica/hwgpe.c b/drivers/acpi/acpica/hwgpe.c index a0e71f34c77a..37bb67ef3232 100644 --- a/drivers/acpi/acpica/hwgpe.c +++ b/drivers/acpi/acpica/hwgpe.c @@ -46,8 +46,13 @@ acpi_status acpi_hw_gpe_read(u64 *value, struct acpi_gpe_address *reg) u32 value32; if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) { +#ifdef ACPI_GPE_USE_LOGICAL_ADDRESSES + *value = (u64)ACPI_GET8(reg->address); + return_ACPI_STATUS(AE_OK); +#else return acpi_os_read_memory((acpi_physical_address)reg->address, value, ACPI_GPE_REGISTER_WIDTH); +#endif } status = acpi_os_read_port((acpi_io_address)reg->address, @@ -76,8 +81,13 @@ acpi_status acpi_hw_gpe_read(u64 *value, struct acpi_gpe_address *reg) acpi_status acpi_hw_gpe_write(u64 value, struct acpi_gpe_address *reg) { if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) { +#ifdef ACPI_GPE_USE_LOGICAL_ADDRESSES + ACPI_SET8(reg->address, value); + return_ACPI_STATUS(AE_OK); +#else return acpi_os_write_memory((acpi_physical_address)reg->address, value, ACPI_GPE_REGISTER_WIDTH); +#endif } return acpi_os_write_port((acpi_io_address)reg->address, (u32)value, From 6915564dc5a8ab831a016e0cd0a8a3c68230287b Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 11 Sep 2020 14:59:35 +0200 Subject: [PATCH 101/265] ACPI: OSL: Change the type of acpi_os_map_generic_address() return value Modify acpi_os_map_generic_address() to return the pointer returned by acpi_os_map_iomem() which represents the logical address corresponding to the struct acpi_generic_address argument passed to it or NULL if that address cannot be obtained (for example, the argument does not represent an address in system memory or it could not be mapped by the OS). Among other things, that will allow the ACPI OS layer to pass the logical addresses of the FADT GPE blocks 0 and 1 to ACPICA going forward. Signed-off-by: Rafael J. Wysocki --- drivers/acpi/apei/apei-base.c | 6 +++++- drivers/acpi/osl.c | 18 +++++++----------- include/acpi/acpi_io.h | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/acpi/apei/apei-base.c b/drivers/acpi/apei/apei-base.c index e358d0046494..552fd9ffaca4 100644 --- a/drivers/acpi/apei/apei-base.c +++ b/drivers/acpi/apei/apei-base.c @@ -632,7 +632,11 @@ int apei_map_generic_address(struct acpi_generic_address *reg) rc = apei_check_gar(reg, &address, &access_bit_width); if (rc) return rc; - return acpi_os_map_generic_address(reg); + + if (!acpi_os_map_generic_address(reg)) + return -ENXIO; + + return 0; } EXPORT_SYMBOL_GPL(apei_map_generic_address); diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c index 4a0b07792233..3a50d8fa310b 100644 --- a/drivers/acpi/osl.c +++ b/drivers/acpi/osl.c @@ -447,24 +447,19 @@ void __ref acpi_os_unmap_memory(void *virt, acpi_size size) } EXPORT_SYMBOL_GPL(acpi_os_unmap_memory); -int acpi_os_map_generic_address(struct acpi_generic_address *gas) +void __iomem *acpi_os_map_generic_address(struct acpi_generic_address *gas) { u64 addr; - void __iomem *virt; if (gas->space_id != ACPI_ADR_SPACE_SYSTEM_MEMORY) - return 0; + return NULL; /* Handle possible alignment issues */ memcpy(&addr, &gas->address, sizeof(addr)); if (!addr || !gas->bit_width) - return -EINVAL; + return NULL; - virt = acpi_os_map_iomem(addr, gas->bit_width / 8); - if (!virt) - return -EIO; - - return 0; + return acpi_os_map_iomem(addr, gas->bit_width / 8); } EXPORT_SYMBOL(acpi_os_map_generic_address); @@ -1756,10 +1751,11 @@ acpi_status __init acpi_os_initialize(void) * Use acpi_os_map_generic_address to pre-map the reset * register if it's in system memory. */ - int rv; + void *rv; rv = acpi_os_map_generic_address(&acpi_gbl_FADT.reset_register); - pr_debug(PREFIX "%s: map reset_reg status %d\n", __func__, rv); + pr_debug(PREFIX "%s: map reset_reg %s\n", __func__, + rv ? "successful" : "failed"); } acpi_os_initialized = true; diff --git a/include/acpi/acpi_io.h b/include/acpi/acpi_io.h index 12d8bd333fe7..027faa8883aa 100644 --- a/include/acpi/acpi_io.h +++ b/include/acpi/acpi_io.h @@ -21,7 +21,7 @@ void __iomem __ref void __ref acpi_os_unmap_iomem(void __iomem *virt, acpi_size size); void __iomem *acpi_os_get_iomem(acpi_physical_address phys, unsigned int size); -int acpi_os_map_generic_address(struct acpi_generic_address *addr); +void __iomem *acpi_os_map_generic_address(struct acpi_generic_address *addr); void acpi_os_unmap_generic_address(struct acpi_generic_address *addr); #endif From 85f94020033f931a5918ab26281b2afbe4d68b73 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 11 Sep 2020 14:59:42 +0200 Subject: [PATCH 102/265] ACPI: OSL: Make ACPICA use logical addresses of GPE blocks Define ACPI_GPE_USE_LOGICAL_ADDRESSES in aclinux.h and modify acpi_os_initialize() to store the logical addresses of the FADT GPE blocks 0 and 1 in acpi_gbl_xgpe0_block_logical_address and acpi_gbl_xgpe1_block_logical_address, respectively, so as to allow ACPICA to use them for accessing GPE registers in system memory, instead of using their physical addresses and looking up the corresponding logical addresses on every access attempt, which is inefficient. Signed-off-by: Rafael J. Wysocki --- drivers/acpi/osl.c | 12 ++++++++++-- include/acpi/platform/aclinux.h | 4 ++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c index 3a50d8fa310b..0418febc5cf2 100644 --- a/drivers/acpi/osl.c +++ b/drivers/acpi/osl.c @@ -1744,8 +1744,12 @@ acpi_status __init acpi_os_initialize(void) { acpi_os_map_generic_address(&acpi_gbl_FADT.xpm1a_event_block); acpi_os_map_generic_address(&acpi_gbl_FADT.xpm1b_event_block); - acpi_os_map_generic_address(&acpi_gbl_FADT.xgpe0_block); - acpi_os_map_generic_address(&acpi_gbl_FADT.xgpe1_block); + + acpi_gbl_xgpe0_block_logical_address = + (unsigned long)acpi_os_map_generic_address(&acpi_gbl_FADT.xgpe0_block); + acpi_gbl_xgpe1_block_logical_address = + (unsigned long)acpi_os_map_generic_address(&acpi_gbl_FADT.xgpe1_block); + if (acpi_gbl_FADT.flags & ACPI_FADT_RESET_REGISTER) { /* * Use acpi_os_map_generic_address to pre-map the reset @@ -1783,8 +1787,12 @@ acpi_status acpi_os_terminate(void) acpi_os_unmap_generic_address(&acpi_gbl_FADT.xgpe1_block); acpi_os_unmap_generic_address(&acpi_gbl_FADT.xgpe0_block); + acpi_gbl_xgpe0_block_logical_address = 0UL; + acpi_gbl_xgpe1_block_logical_address = 0UL; + acpi_os_unmap_generic_address(&acpi_gbl_FADT.xpm1b_event_block); acpi_os_unmap_generic_address(&acpi_gbl_FADT.xpm1a_event_block); + if (acpi_gbl_FADT.flags & ACPI_FADT_RESET_REGISTER) acpi_os_unmap_generic_address(&acpi_gbl_FADT.reset_register); diff --git a/include/acpi/platform/aclinux.h b/include/acpi/platform/aclinux.h index 987e2af7c335..4151c76141fa 100644 --- a/include/acpi/platform/aclinux.h +++ b/include/acpi/platform/aclinux.h @@ -118,6 +118,10 @@ #define USE_NATIVE_ALLOCATE_ZEROED +/* Use logical addresses for accessing GPE registers in system memory */ + +#define ACPI_GPE_USE_LOGICAL_ADDRESSES + /* * Overrides for in-kernel ACPICA */ From 5f155515d37351f90dbb1c8a5a5cae13190c8564 Mon Sep 17 00:00:00 2001 From: Wang Qing Date: Thu, 13 Aug 2020 10:49:10 +0800 Subject: [PATCH 103/265] ACPI: NFIT: Use kobj_to_dev() instead Use kobj_to_dev() instead of container_of() Signed-off-by: Wang Qing Acked-by: Vishal Verma [ rjw: Subject edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/nfit/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index 26dd208a0d63..1904ae88a11c 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -1389,7 +1389,7 @@ static bool ars_supported(struct nvdimm_bus *nvdimm_bus) static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n) { - struct device *dev = container_of(kobj, struct device, kobj); + struct device *dev = kobj_to_dev(kobj); struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); if (a == &dev_attr_scrub.attr) @@ -1679,7 +1679,7 @@ static struct attribute *acpi_nfit_dimm_attributes[] = { static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj, struct attribute *a, int n) { - struct device *dev = container_of(kobj, struct device, kobj); + struct device *dev = kobj_to_dev(kobj); struct nvdimm *nvdimm = to_nvdimm(dev); struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); From fa870509d9ecd408714d8888568ccc9f2c52af2a Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 14 Aug 2020 16:27:25 +0300 Subject: [PATCH 104/265] ACPI / PMIC: Split out Kconfig and Makefile specific for ACPI PMIC It's a bit better to maintain and allows to avoid mistakes in the future with PMIC OpRegion drivers, if we split out Kconfig and Makefile for ACPI PMIC to its own folder. Signed-off-by: Andy Shevchenko Reviewed-by: Mika Westerberg Signed-off-by: Rafael J. Wysocki --- drivers/acpi/Kconfig | 51 ++------------------------------------ drivers/acpi/Makefile | 9 +------ drivers/acpi/pmic/Kconfig | 51 ++++++++++++++++++++++++++++++++++++++ drivers/acpi/pmic/Makefile | 9 +++++++ 4 files changed, 63 insertions(+), 57 deletions(-) create mode 100644 drivers/acpi/pmic/Kconfig create mode 100644 drivers/acpi/pmic/Makefile diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index 7540a5179a47..6fb9453396dc 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -504,55 +504,6 @@ config ACPI_EXTLOG config ACPI_ADXL bool -menuconfig PMIC_OPREGION - bool "PMIC (Power Management Integrated Circuit) operation region support" - help - Select this option to enable support for ACPI operation - region of the PMIC chip. The operation region can be used - to control power rails and sensor reading/writing on the - PMIC chip. - -if PMIC_OPREGION -config BYTCRC_PMIC_OPREGION - bool "ACPI operation region support for Bay Trail Crystal Cove PMIC" - depends on INTEL_SOC_PMIC - help - This config adds ACPI operation region support for the Bay Trail - version of the Crystal Cove PMIC. - -config CHTCRC_PMIC_OPREGION - bool "ACPI operation region support for Cherry Trail Crystal Cove PMIC" - depends on INTEL_SOC_PMIC - help - This config adds ACPI operation region support for the Cherry Trail - version of the Crystal Cove PMIC. - -config XPOWER_PMIC_OPREGION - bool "ACPI operation region support for XPower AXP288 PMIC" - depends on MFD_AXP20X_I2C && IOSF_MBI=y - help - This config adds ACPI operation region support for XPower AXP288 PMIC. - -config BXT_WC_PMIC_OPREGION - bool "ACPI operation region support for BXT WhiskeyCove PMIC" - depends on INTEL_SOC_PMIC_BXTWC - help - This config adds ACPI operation region support for BXT WhiskeyCove PMIC. - -config CHT_WC_PMIC_OPREGION - bool "ACPI operation region support for CHT Whiskey Cove PMIC" - depends on INTEL_SOC_PMIC_CHTWC - help - This config adds ACPI operation region support for CHT Whiskey Cove PMIC. - -config CHT_DC_TI_PMIC_OPREGION - bool "ACPI operation region support for Dollar Cove TI PMIC" - depends on INTEL_SOC_PMIC_CHTDC_TI - help - This config adds ACPI operation region support for Dollar Cove TI PMIC. - -endif - config ACPI_CONFIGFS tristate "ACPI configfs support" select CONFIGFS_FS @@ -568,6 +519,8 @@ config ACPI_PPTT bool endif +source "drivers/acpi/pmic/Kconfig" + config TPS68470_PMIC_OPREGION bool "ACPI operation region support for TPS68470 PMIC" depends on MFD_TPS68470 diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile index 9a957544e357..44929d248bfa 100644 --- a/drivers/acpi/Makefile +++ b/drivers/acpi/Makefile @@ -107,16 +107,9 @@ obj-$(CONFIG_ACPI_APEI) += apei/ obj-$(CONFIG_ACPI_EXTLOG) += acpi_extlog.o -obj-$(CONFIG_PMIC_OPREGION) += pmic/intel_pmic.o -obj-$(CONFIG_BYTCRC_PMIC_OPREGION) += pmic/intel_pmic_bytcrc.o -obj-$(CONFIG_CHTCRC_PMIC_OPREGION) += pmic/intel_pmic_chtcrc.o -obj-$(CONFIG_XPOWER_PMIC_OPREGION) += pmic/intel_pmic_xpower.o -obj-$(CONFIG_BXT_WC_PMIC_OPREGION) += pmic/intel_pmic_bxtwc.o -obj-$(CONFIG_CHT_WC_PMIC_OPREGION) += pmic/intel_pmic_chtwc.o -obj-$(CONFIG_CHT_DC_TI_PMIC_OPREGION) += pmic/intel_pmic_chtdc_ti.o - obj-$(CONFIG_ACPI_CONFIGFS) += acpi_configfs.o +obj-y += pmic/ obj-$(CONFIG_TPS68470_PMIC_OPREGION) += pmic/tps68470_pmic.o video-objs += acpi_video.o video_detect.o diff --git a/drivers/acpi/pmic/Kconfig b/drivers/acpi/pmic/Kconfig new file mode 100644 index 000000000000..357d1a846e68 --- /dev/null +++ b/drivers/acpi/pmic/Kconfig @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-2.0 + +menuconfig PMIC_OPREGION + bool "PMIC (Power Management Integrated Circuit) operation region support" + help + Select this option to enable support for ACPI operation + region of the PMIC chip. The operation region can be used + to control power rails and sensor reading/writing on the + PMIC chip. + +if PMIC_OPREGION + +config BYTCRC_PMIC_OPREGION + bool "ACPI operation region support for Bay Trail Crystal Cove PMIC" + depends on INTEL_SOC_PMIC + help + This config adds ACPI operation region support for the Bay Trail + version of the Crystal Cove PMIC. + +config CHTCRC_PMIC_OPREGION + bool "ACPI operation region support for Cherry Trail Crystal Cove PMIC" + depends on INTEL_SOC_PMIC + help + This config adds ACPI operation region support for the Cherry Trail + version of the Crystal Cove PMIC. + +config XPOWER_PMIC_OPREGION + bool "ACPI operation region support for XPower AXP288 PMIC" + depends on MFD_AXP20X_I2C && IOSF_MBI=y + help + This config adds ACPI operation region support for XPower AXP288 PMIC. + +config BXT_WC_PMIC_OPREGION + bool "ACPI operation region support for BXT WhiskeyCove PMIC" + depends on INTEL_SOC_PMIC_BXTWC + help + This config adds ACPI operation region support for BXT WhiskeyCove PMIC. + +config CHT_WC_PMIC_OPREGION + bool "ACPI operation region support for CHT Whiskey Cove PMIC" + depends on INTEL_SOC_PMIC_CHTWC + help + This config adds ACPI operation region support for CHT Whiskey Cove PMIC. + +config CHT_DC_TI_PMIC_OPREGION + bool "ACPI operation region support for Dollar Cove TI PMIC" + depends on INTEL_SOC_PMIC_CHTDC_TI + help + This config adds ACPI operation region support for Dollar Cove TI PMIC. + +endif # PMIC_OPREGION diff --git a/drivers/acpi/pmic/Makefile b/drivers/acpi/pmic/Makefile new file mode 100644 index 000000000000..773c267420bc --- /dev/null +++ b/drivers/acpi/pmic/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_PMIC_OPREGION) += intel_pmic.o +obj-$(CONFIG_BYTCRC_PMIC_OPREGION) += intel_pmic_bytcrc.o +obj-$(CONFIG_CHTCRC_PMIC_OPREGION) += intel_pmic_chtcrc.o +obj-$(CONFIG_XPOWER_PMIC_OPREGION) += intel_pmic_xpower.o +obj-$(CONFIG_BXT_WC_PMIC_OPREGION) += intel_pmic_bxtwc.o +obj-$(CONFIG_CHT_WC_PMIC_OPREGION) += intel_pmic_chtwc.o +obj-$(CONFIG_CHT_DC_TI_PMIC_OPREGION) += intel_pmic_chtdc_ti.o From e410c43b66d52dde6a4b8554dc85a9dc06e57937 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 14 Aug 2020 16:27:26 +0300 Subject: [PATCH 105/265] ACPI / PMIC: Move TPS68470 OpRegion driver to drivers/acpi/pmic/ It is revealed now that TPS68470 OpRegion driver has been added in slightly different scope. Let's move it to the drivers/acpi/pmic/ folder for sake of the unification. Signed-off-by: Andy Shevchenko Reviewed-by: Mika Westerberg Signed-off-by: Rafael J. Wysocki --- drivers/acpi/Kconfig | 16 ---------------- drivers/acpi/Makefile | 1 - drivers/acpi/pmic/Kconfig | 16 ++++++++++++++++ drivers/acpi/pmic/Makefile | 1 + 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index 6fb9453396dc..edf1558c1105 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -521,22 +521,6 @@ endif source "drivers/acpi/pmic/Kconfig" -config TPS68470_PMIC_OPREGION - bool "ACPI operation region support for TPS68470 PMIC" - depends on MFD_TPS68470 - help - This config adds ACPI operation region support for TI TPS68470 PMIC. - TPS68470 device is an advanced power management unit that powers - a Compact Camera Module (CCM), generates clocks for image sensors, - drives a dual LED for flash and incorporates two LED drivers for - general purpose indicators. - This driver enables ACPI operation region support control voltage - regulators and clocks. - - This option is a bool as it provides an ACPI operation - region, which must be available before any of the devices - using this, are probed. - endif # ACPI config X86_PM_TIMER diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile index 44929d248bfa..44e412506317 100644 --- a/drivers/acpi/Makefile +++ b/drivers/acpi/Makefile @@ -110,7 +110,6 @@ obj-$(CONFIG_ACPI_EXTLOG) += acpi_extlog.o obj-$(CONFIG_ACPI_CONFIGFS) += acpi_configfs.o obj-y += pmic/ -obj-$(CONFIG_TPS68470_PMIC_OPREGION) += pmic/tps68470_pmic.o video-objs += acpi_video.o video_detect.o obj-y += dptf/ diff --git a/drivers/acpi/pmic/Kconfig b/drivers/acpi/pmic/Kconfig index 357d1a846e68..56bbcb2ce61b 100644 --- a/drivers/acpi/pmic/Kconfig +++ b/drivers/acpi/pmic/Kconfig @@ -49,3 +49,19 @@ config CHT_DC_TI_PMIC_OPREGION This config adds ACPI operation region support for Dollar Cove TI PMIC. endif # PMIC_OPREGION + +config TPS68470_PMIC_OPREGION + bool "ACPI operation region support for TPS68470 PMIC" + depends on MFD_TPS68470 + help + This config adds ACPI operation region support for TI TPS68470 PMIC. + TPS68470 device is an advanced power management unit that powers + a Compact Camera Module (CCM), generates clocks for image sensors, + drives a dual LED for flash and incorporates two LED drivers for + general purpose indicators. + This driver enables ACPI operation region support control voltage + regulators and clocks. + + This option is a bool as it provides an ACPI operation + region, which must be available before any of the devices + using this, are probed. diff --git a/drivers/acpi/pmic/Makefile b/drivers/acpi/pmic/Makefile index 773c267420bc..cd072c64920c 100644 --- a/drivers/acpi/pmic/Makefile +++ b/drivers/acpi/pmic/Makefile @@ -7,3 +7,4 @@ obj-$(CONFIG_XPOWER_PMIC_OPREGION) += intel_pmic_xpower.o obj-$(CONFIG_BXT_WC_PMIC_OPREGION) += intel_pmic_bxtwc.o obj-$(CONFIG_CHT_WC_PMIC_OPREGION) += intel_pmic_chtwc.o obj-$(CONFIG_CHT_DC_TI_PMIC_OPREGION) += intel_pmic_chtdc_ti.o +obj-$(CONFIG_TPS68470_PMIC_OPREGION) += tps68470_pmic.o From 90d46d71cce279d878793a0ed4b326b4027aca6c Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 1 Sep 2020 15:07:09 +0530 Subject: [PATCH 106/265] opp: Handle multiple calls for same OPP table in _of_add_opp_table_v1() Until now for V1 OPP bindings we used to call dev_pm_opp_of_cpumask_add_table() first and then dev_pm_opp_set_sharing_cpus() in the cpufreq-dt driver. A later patch will though update the cpufreq-dt driver to optimize the code a bit and we will call dev_pm_opp_set_sharing_cpus() first followed by dev_pm_opp_of_cpumask_add_table(), which doesn't work well today as it tries to re parse the OPP entries. This should work nevertheless for V1 bindings as the same works for V2 bindings. Adapt the same approach from V2 bindings and fix this. Reported-by: Marek Szyprowski Tested-by: Marek Szyprowski Signed-off-by: Viresh Kumar --- drivers/opp/of.c | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/drivers/opp/of.c b/drivers/opp/of.c index d8b623cc015a..01a9f1ff5bb0 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -886,11 +886,25 @@ static int _of_add_opp_table_v1(struct device *dev, struct opp_table *opp_table) const __be32 *val; int nr, ret = 0; + mutex_lock(&opp_table->lock); + if (opp_table->parsed_static_opps) { + opp_table->parsed_static_opps++; + mutex_unlock(&opp_table->lock); + return 0; + } + + opp_table->parsed_static_opps = 1; + mutex_unlock(&opp_table->lock); + prop = of_find_property(dev->of_node, "operating-points", NULL); - if (!prop) - return -ENODEV; - if (!prop->value) - return -ENODATA; + if (!prop) { + ret = -ENODEV; + goto remove_static_opp; + } + if (!prop->value) { + ret = -ENODATA; + goto remove_static_opp; + } /* * Each OPP is a set of tuples consisting of frequency and @@ -899,13 +913,10 @@ static int _of_add_opp_table_v1(struct device *dev, struct opp_table *opp_table) nr = prop->length / sizeof(u32); if (nr % 2) { dev_err(dev, "%s: Invalid OPP table\n", __func__); - return -EINVAL; + ret = -EINVAL; + goto remove_static_opp; } - mutex_lock(&opp_table->lock); - opp_table->parsed_static_opps = 1; - mutex_unlock(&opp_table->lock); - val = prop->value; while (nr) { unsigned long freq = be32_to_cpup(val++) * 1000; @@ -915,12 +926,14 @@ static int _of_add_opp_table_v1(struct device *dev, struct opp_table *opp_table) if (ret) { dev_err(dev, "%s: Failed to add OPP %ld (%d)\n", __func__, freq, ret); - _opp_remove_all_static(opp_table); - return ret; + goto remove_static_opp; } nr -= 2; } +remove_static_opp: + _opp_remove_all_static(opp_table); + return ret; } From dc279ac6e5b4e06ec9c15b82e30e8bf2576b14f9 Mon Sep 17 00:00:00 2001 From: Stephan Gerhold Date: Mon, 27 Jul 2020 11:30:47 +0200 Subject: [PATCH 107/265] cpufreq: dt: Refactor initialization to handle probe deferral properly cpufreq-dt is currently unable to handle -EPROBE_DEFER properly because the error code is not propagated for the cpufreq_driver->init() callback. Instead, it attempts to avoid the situation by temporarily requesting all resources within resources_available() and releasing them again immediately after. This has several disadvantages: - Whenever we add something like interconnect handling to the OPP core we need to patch cpufreq-dt to request these resources early. - resources_available() is only run for CPU0, but other clusters may eventually depend on other resources that are not available yet. (See FIXME comment removed by this commit...) - All resources need to be looked up several times. Now that the OPP core can propagate -EPROBE_DEFER during initialization, it would be nice to avoid all that trouble and just propagate its error code when necessary. This commit refactors the cpufreq-dt driver to initialize private_data before registering the cpufreq driver. We do this by iterating over all possible CPUs and ensure that all resources are initialized: 1. dev_pm_opp_get_opp_table() ensures the OPP table is allocated and initialized with clock and interconnects. 2. dev_pm_opp_set_regulators() requests the regulators and assigns them to the OPP table. 3. We call dev_pm_opp_of_get_sharing_cpus() early so that we only initialize the OPP table once for each shared policy. With these changes, we actually end up saving a few lines of code, the resources are no longer looked up multiple times and everything should be much more robust. Signed-off-by: Stephan Gerhold [ Viresh: Use list_head structure for maintaining the list and minor changes ] Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq-dt.c | 288 +++++++++++++++++------------------ 1 file changed, 144 insertions(+), 144 deletions(-) diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index 944d7b45afe9..f0c2d3876dec 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -24,18 +25,35 @@ #include "cpufreq-dt.h" struct private_data { - struct opp_table *opp_table; + struct list_head node; + + cpumask_var_t cpus; struct device *cpu_dev; - const char *reg_name; + struct opp_table *opp_table; + struct opp_table *reg_opp_table; bool have_static_opps; }; +static LIST_HEAD(priv_list); + static struct freq_attr *cpufreq_dt_attr[] = { &cpufreq_freq_attr_scaling_available_freqs, NULL, /* Extra space for boost-attr if required */ NULL, }; +static struct private_data *cpufreq_dt_find_data(int cpu) +{ + struct private_data *priv; + + list_for_each_entry(priv, &priv_list, node) { + if (cpumask_test_cpu(cpu, priv->cpus)) + return priv; + } + + return NULL; +} + static int set_target(struct cpufreq_policy *policy, unsigned int index) { struct private_data *priv = policy->driver_data; @@ -90,83 +108,24 @@ node_put: return name; } -static int resources_available(void) -{ - struct device *cpu_dev; - struct regulator *cpu_reg; - struct clk *cpu_clk; - int ret = 0; - const char *name; - - cpu_dev = get_cpu_device(0); - if (!cpu_dev) { - pr_err("failed to get cpu0 device\n"); - return -ENODEV; - } - - cpu_clk = clk_get(cpu_dev, NULL); - ret = PTR_ERR_OR_ZERO(cpu_clk); - if (ret) { - /* - * If cpu's clk node is present, but clock is not yet - * registered, we should try defering probe. - */ - if (ret == -EPROBE_DEFER) - dev_dbg(cpu_dev, "clock not ready, retry\n"); - else - dev_err(cpu_dev, "failed to get clock: %d\n", ret); - - return ret; - } - - clk_put(cpu_clk); - - ret = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL); - if (ret) - return ret; - - name = find_supply_name(cpu_dev); - /* Platform doesn't require regulator */ - if (!name) - return 0; - - cpu_reg = regulator_get_optional(cpu_dev, name); - ret = PTR_ERR_OR_ZERO(cpu_reg); - if (ret) { - /* - * If cpu's regulator supply node is present, but regulator is - * not yet registered, we should try defering probe. - */ - if (ret == -EPROBE_DEFER) - dev_dbg(cpu_dev, "cpu0 regulator not ready, retry\n"); - else - dev_dbg(cpu_dev, "no regulator for cpu0: %d\n", ret); - - return ret; - } - - regulator_put(cpu_reg); - return 0; -} - static int cpufreq_init(struct cpufreq_policy *policy) { struct cpufreq_frequency_table *freq_table; - struct opp_table *opp_table = NULL; struct private_data *priv; struct device *cpu_dev; struct clk *cpu_clk; unsigned int transition_latency; - bool fallback = false; - const char *name; int ret; - cpu_dev = get_cpu_device(policy->cpu); - if (!cpu_dev) { - pr_err("failed to get cpu%d device\n", policy->cpu); + priv = cpufreq_dt_find_data(policy->cpu); + if (!priv) { + pr_err("failed to find data for cpu%d\n", policy->cpu); return -ENODEV; } + cpu_dev = priv->cpu_dev; + cpumask_copy(policy->cpus, priv->cpus); + cpu_clk = clk_get(cpu_dev, NULL); if (IS_ERR(cpu_clk)) { ret = PTR_ERR(cpu_clk); @@ -174,45 +133,6 @@ static int cpufreq_init(struct cpufreq_policy *policy) return ret; } - /* Get OPP-sharing information from "operating-points-v2" bindings */ - ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, policy->cpus); - if (ret) { - if (ret != -ENOENT) - goto out_put_clk; - - /* - * operating-points-v2 not supported, fallback to old method of - * finding shared-OPPs for backward compatibility if the - * platform hasn't set sharing CPUs. - */ - if (dev_pm_opp_get_sharing_cpus(cpu_dev, policy->cpus)) - fallback = true; - } - - /* - * OPP layer will be taking care of regulators now, but it needs to know - * the name of the regulator first. - */ - name = find_supply_name(cpu_dev); - if (name) { - opp_table = dev_pm_opp_set_regulators(cpu_dev, &name, 1); - if (IS_ERR(opp_table)) { - ret = PTR_ERR(opp_table); - dev_err(cpu_dev, "Failed to set regulator for cpu%d: %d\n", - policy->cpu, ret); - goto out_put_clk; - } - } - - priv = kzalloc(sizeof(*priv), GFP_KERNEL); - if (!priv) { - ret = -ENOMEM; - goto out_put_regulator; - } - - priv->reg_name = name; - priv->opp_table = opp_table; - /* * Initialize OPP tables for all policy->cpus. They will be shared by * all CPUs which have marked their CPUs shared with OPP bindings. @@ -232,31 +152,17 @@ static int cpufreq_init(struct cpufreq_policy *policy) */ ret = dev_pm_opp_get_opp_count(cpu_dev); if (ret <= 0) { - dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n"); - ret = -EPROBE_DEFER; + dev_err(cpu_dev, "OPP table can't be empty\n"); + ret = -ENODEV; goto out_free_opp; } - if (fallback) { - cpumask_setall(policy->cpus); - - /* - * OPP tables are initialized only for policy->cpu, do it for - * others as well. - */ - ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); - if (ret) - dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", - __func__, ret); - } - ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); if (ret) { dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); goto out_free_opp; } - priv->cpu_dev = cpu_dev; policy->driver_data = priv; policy->clk = cpu_clk; policy->freq_table = freq_table; @@ -288,11 +194,6 @@ out_free_cpufreq_table: out_free_opp: if (priv->have_static_opps) dev_pm_opp_of_cpumask_remove_table(policy->cpus); - kfree(priv); -out_put_regulator: - if (name) - dev_pm_opp_put_regulators(opp_table); -out_put_clk: clk_put(cpu_clk); return ret; @@ -320,12 +221,7 @@ static int cpufreq_exit(struct cpufreq_policy *policy) dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); if (priv->have_static_opps) dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); - if (priv->reg_name) - dev_pm_opp_put_regulators(priv->opp_table); - clk_put(policy->clk); - kfree(priv); - return 0; } @@ -344,21 +240,119 @@ static struct cpufreq_driver dt_cpufreq_driver = { .suspend = cpufreq_generic_suspend, }; +static int dt_cpufreq_early_init(struct device *dev, int cpu) +{ + struct private_data *priv; + struct device *cpu_dev; + const char *reg_name; + int ret; + + /* Check if this CPU is already covered by some other policy */ + if (cpufreq_dt_find_data(cpu)) + return 0; + + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) + return -EPROBE_DEFER; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + if (!alloc_cpumask_var(&priv->cpus, GFP_KERNEL)) + return -ENOMEM; + + priv->cpu_dev = cpu_dev; + + /* Try to get OPP table early to ensure resources are available */ + priv->opp_table = dev_pm_opp_get_opp_table(cpu_dev); + if (IS_ERR(priv->opp_table)) { + ret = PTR_ERR(priv->opp_table); + if (ret != -EPROBE_DEFER) + dev_err(cpu_dev, "failed to get OPP table: %d\n", ret); + goto free_cpumask; + } + + /* + * OPP layer will be taking care of regulators now, but it needs to know + * the name of the regulator first. + */ + reg_name = find_supply_name(cpu_dev); + if (reg_name) { + priv->reg_opp_table = dev_pm_opp_set_regulators(cpu_dev, + ®_name, 1); + if (IS_ERR(priv->reg_opp_table)) { + ret = PTR_ERR(priv->reg_opp_table); + if (ret != -EPROBE_DEFER) + dev_err(cpu_dev, "failed to set regulators: %d\n", + ret); + goto put_table; + } + } + + /* Find OPP sharing information so we can fill pri->cpus here */ + /* Get OPP-sharing information from "operating-points-v2" bindings */ + ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, priv->cpus); + if (ret) { + if (ret != -ENOENT) + goto put_reg; + + /* + * operating-points-v2 not supported, fallback to all CPUs share + * OPP for backward compatibility if the platform hasn't set + * sharing CPUs. + */ + if (dev_pm_opp_get_sharing_cpus(cpu_dev, priv->cpus)) { + cpumask_setall(priv->cpus); + + /* + * OPP tables are initialized only for cpu, do it for + * others as well. + */ + ret = dev_pm_opp_set_sharing_cpus(cpu_dev, priv->cpus); + if (ret) + dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", + __func__, ret); + } + } + + list_add(&priv->node, &priv_list); + return 0; + +put_reg: + if (priv->reg_opp_table) + dev_pm_opp_put_regulators(priv->reg_opp_table); +put_table: + dev_pm_opp_put_opp_table(priv->opp_table); +free_cpumask: + free_cpumask_var(priv->cpus); + return ret; +} + +static void dt_cpufreq_release(void) +{ + struct private_data *priv, *tmp; + + list_for_each_entry_safe(priv, tmp, &priv_list, node) { + if (priv->reg_opp_table) + dev_pm_opp_put_regulators(priv->reg_opp_table); + dev_pm_opp_put_opp_table(priv->opp_table); + free_cpumask_var(priv->cpus); + list_del(&priv->node); + } +} + static int dt_cpufreq_probe(struct platform_device *pdev) { struct cpufreq_dt_platform_data *data = dev_get_platdata(&pdev->dev); - int ret; + int ret, cpu; - /* - * All per-cluster (CPUs sharing clock/voltages) initialization is done - * from ->init(). In probe(), we just need to make sure that clk and - * regulators are available. Else defer probe and retry. - * - * FIXME: Is checking this only for CPU0 sufficient ? - */ - ret = resources_available(); - if (ret) - return ret; + /* Request resources early so we can return in case of -EPROBE_DEFER */ + for_each_possible_cpu(cpu) { + ret = dt_cpufreq_early_init(&pdev->dev, cpu); + if (ret) + goto err; + } if (data) { if (data->have_governor_per_policy) @@ -374,15 +368,21 @@ static int dt_cpufreq_probe(struct platform_device *pdev) } ret = cpufreq_register_driver(&dt_cpufreq_driver); - if (ret) + if (ret) { dev_err(&pdev->dev, "failed register driver: %d\n", ret); + goto err; + } + return 0; +err: + dt_cpufreq_release(); return ret; } static int dt_cpufreq_remove(struct platform_device *pdev) { cpufreq_unregister_driver(&dt_cpufreq_driver); + dt_cpufreq_release(); return 0; } From 475ac8ead803e8c3a8f492e3129b1a19d7e2efd7 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 27 Aug 2020 15:06:43 +0530 Subject: [PATCH 108/265] opp: Drop unnecessary check from dev_pm_opp_attach_genpd() Since commit c0ab9e0812da ("opp: Allocate genpd_virt_devs from dev_pm_opp_attach_genpd()"), the allocation of the virtual devices is moved to dev_pm_opp_attach_genpd() and this check isn't required anymore as it will always fail. Drop it. Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 8c69a764d0a4..fa97a875eb47 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2019,12 +2019,6 @@ struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, goto err; } - if (opp_table->genpd_virt_devs[index]) { - dev_err(dev, "Genpd virtual device already set %s\n", - *name); - goto err; - } - virt_dev = dev_pm_domain_attach_by_name(dev, *name); if (IS_ERR(virt_dev)) { ret = PTR_ERR(virt_dev); From 60cdeae0d627eca4ae616e3097a1f00ac9f3d704 Mon Sep 17 00:00:00 2001 From: Stephan Gerhold Date: Thu, 30 Jul 2020 10:01:44 +0200 Subject: [PATCH 109/265] opp: Reduce code duplication in _set_required_opps() Move call to dev_pm_genpd_set_performance_state() to a separate function so we can avoid duplicating the code for the single and multiple genpd case. Signed-off-by: Stephan Gerhold [ Viresh: Validate virtual device before use ] Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/drivers/opp/core.c b/drivers/opp/core.c index fa97a875eb47..79c873344895 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -779,6 +779,24 @@ static int _set_opp_custom(const struct opp_table *opp_table, return opp_table->set_opp(data); } +static int _set_required_opp(struct device *dev, struct device *pd_dev, + struct dev_pm_opp *opp, int i) +{ + unsigned int pstate = likely(opp) ? opp->required_opps[i]->pstate : 0; + int ret; + + if (!pd_dev) + return 0; + + ret = dev_pm_genpd_set_performance_state(pd_dev, pstate); + if (ret) { + dev_err(dev, "Failed to set performance rate of %s: %d (%d)\n", + dev_name(pd_dev), pstate, ret); + } + + return ret; +} + /* This is only called for PM domain for now */ static int _set_required_opps(struct device *dev, struct opp_table *opp_table, @@ -786,22 +804,14 @@ static int _set_required_opps(struct device *dev, { struct opp_table **required_opp_tables = opp_table->required_opp_tables; struct device **genpd_virt_devs = opp_table->genpd_virt_devs; - unsigned int pstate; int i, ret = 0; if (!required_opp_tables) return 0; /* Single genpd case */ - if (!genpd_virt_devs) { - pstate = likely(opp) ? opp->required_opps[0]->pstate : 0; - ret = dev_pm_genpd_set_performance_state(dev, pstate); - if (ret) { - dev_err(dev, "Failed to set performance state of %s: %d (%d)\n", - dev_name(dev), pstate, ret); - } - return ret; - } + if (!genpd_virt_devs) + return _set_required_opp(dev, dev, opp, 0); /* Multiple genpd case */ @@ -810,19 +820,10 @@ static int _set_required_opps(struct device *dev, * after it is freed from another thread. */ mutex_lock(&opp_table->genpd_virt_dev_lock); - for (i = 0; i < opp_table->required_opp_count; i++) { - pstate = likely(opp) ? opp->required_opps[i]->pstate : 0; - - if (!genpd_virt_devs[i]) - continue; - - ret = dev_pm_genpd_set_performance_state(genpd_virt_devs[i], pstate); - if (ret) { - dev_err(dev, "Failed to set performance rate of %s: %d (%d)\n", - dev_name(genpd_virt_devs[i]), pstate, ret); + ret = _set_required_opp(dev, genpd_virt_devs[i], opp, i); + if (ret) break; - } } mutex_unlock(&opp_table->genpd_virt_dev_lock); From 2c59138c22f17c1da027d3c90dbcdd6995c77414 Mon Sep 17 00:00:00 2001 From: Stephan Gerhold Date: Thu, 30 Jul 2020 10:01:45 +0200 Subject: [PATCH 110/265] opp: Set required OPPs in reverse order when scaling down The OPP core already has well-defined semantics to ensure required OPPs/regulators are set before/after the frequency change, depending on if we scale up or down. Similar requirements might exist for the order of required OPPs when multiple power domains need to be scaled for a frequency change. For example, on Qualcomm platforms using CPR (Core Power Reduction), we need to scale the VDDMX and CPR power domain. When scaling up, MX should be scaled up before CPR. When scaling down, CPR should be scaled down before MX. In general, if there are multiple "required-opps" in the device tree I would expect that the order is either irrelevant, or there is some dependency between the power domains. In that case, the power domains should be scaled down in reverse order. This commit updates _set_required_opps() to set required OPPs in reverse order when scaling down. Signed-off-by: Stephan Gerhold [ Viresh: Fix rebase conflict and minor rearrangement of the code ] Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 79c873344895..000d0fcb4680 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -800,7 +800,7 @@ static int _set_required_opp(struct device *dev, struct device *pd_dev, /* This is only called for PM domain for now */ static int _set_required_opps(struct device *dev, struct opp_table *opp_table, - struct dev_pm_opp *opp) + struct dev_pm_opp *opp, bool up) { struct opp_table **required_opp_tables = opp_table->required_opp_tables; struct device **genpd_virt_devs = opp_table->genpd_virt_devs; @@ -820,11 +820,22 @@ static int _set_required_opps(struct device *dev, * after it is freed from another thread. */ mutex_lock(&opp_table->genpd_virt_dev_lock); - for (i = 0; i < opp_table->required_opp_count; i++) { - ret = _set_required_opp(dev, genpd_virt_devs[i], opp, i); - if (ret) - break; + + /* Scaling up? Set required OPPs in normal order, else reverse */ + if (up) { + for (i = 0; i < opp_table->required_opp_count; i++) { + ret = _set_required_opp(dev, genpd_virt_devs[i], opp, i); + if (ret) + break; + } + } else { + for (i = opp_table->required_opp_count - 1; i >= 0; i--) { + ret = _set_required_opp(dev, genpd_virt_devs[i], opp, i); + if (ret) + break; + } } + mutex_unlock(&opp_table->genpd_virt_dev_lock); return ret; @@ -883,7 +894,7 @@ static int _opp_set_rate_zero(struct device *dev, struct opp_table *opp_table) if (opp_table->regulators) regulator_disable(opp_table->regulators[0]); - ret = _set_required_opps(dev, opp_table, NULL); + ret = _set_required_opps(dev, opp_table, NULL, false); opp_table->enabled = false; return ret; @@ -974,7 +985,7 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) /* Scaling up? Configure required OPPs before frequency */ if (freq >= old_freq) { - ret = _set_required_opps(dev, opp_table, opp); + ret = _set_required_opps(dev, opp_table, opp, true); if (ret) goto put_opp; } @@ -994,7 +1005,7 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) /* Scaling down? Configure required OPPs after frequency */ if (!ret && freq < old_freq) { - ret = _set_required_opps(dev, opp_table, opp); + ret = _set_required_opps(dev, opp_table, opp, false); if (ret) dev_err(dev, "Failed to set required opps: %d\n", ret); } From 4461269572e6fc216b63e2254c83035d1bc21eb7 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 26 Aug 2020 15:09:16 +0530 Subject: [PATCH 111/265] dt-bindings: opp: Allow opp-supported-hw to contain multiple versions A single list of versions for a hierarchy of hardware levels is not sufficient in some cases. For example, if the hardware version has two levels, i.e. X.Y and we want an OPP to support only version 2.1 and 1.2, we will set the property as: opp-supported-hw = <0x00000003 0x00000003>; What this also does is enable hardware versions 2.2 and 1.1, which we don't want. Extend the property to accept multiple versions, so we can define the property as: opp-supported-hw = <0x00000002 0x00000001>, <0x00000001 0x00000002>; While at it, also reword the property description. Reported-by: Stephan Gerhold Signed-off-by: Viresh Kumar --- Documentation/devicetree/bindings/opp/opp.txt | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/Documentation/devicetree/bindings/opp/opp.txt b/Documentation/devicetree/bindings/opp/opp.txt index 9d16d417e9be..9847dfeeffcb 100644 --- a/Documentation/devicetree/bindings/opp/opp.txt +++ b/Documentation/devicetree/bindings/opp/opp.txt @@ -154,25 +154,27 @@ Optional properties: - opp-suspend: Marks the OPP to be used during device suspend. If multiple OPPs in the table have this, the OPP with highest opp-hz will be used. -- opp-supported-hw: This enables us to select only a subset of OPPs from the - larger OPP table, based on what version of the hardware we are running on. We - still can't have multiple nodes with the same opp-hz value in OPP table. +- opp-supported-hw: This property allows a platform to enable only a subset of + the OPPs from the larger set present in the OPP table, based on the current + version of the hardware (already known to the operating system). - It's a user defined array containing a hierarchy of hardware version numbers, - supported by the OPP. For example: a platform with hierarchy of three levels - of versions (A, B and C), this field should be like , where X - corresponds to Version hierarchy A, Y corresponds to version hierarchy B and Z - corresponds to version hierarchy C. + Each block present in the array of blocks in this property, represents a + sub-group of hardware versions supported by the OPP. i.e. , + , etc. The OPP will be enabled if _any_ of these sub-groups match + the hardware's version. - Each level of hierarchy is represented by a 32 bit value, and so there can be - only 32 different supported version per hierarchy. i.e. 1 bit per version. A - value of 0xFFFFFFFF will enable the OPP for all versions for that hierarchy - level. And a value of 0x00000000 will disable the OPP completely, and so we - never want that to happen. + Each sub-group is a platform defined array representing the hierarchy of + hardware versions supported by the platform. For a platform with three + hierarchical levels of version (X.Y.Z), this field shall look like - If 32 values aren't sufficient for a version hierarchy, than that version - hierarchy can be contained in multiple 32 bit values. i.e. in the - above example, Z1 & Z2 refer to the version hierarchy Z. + opp-supported-hw = , , . + + Each level (eg. X1) in version hierarchy is represented by a 32 bit value, one + bit per version and so there can be maximum 32 versions per level. Logical AND + (&) operation is performed for each level with the hardware's level version + and a non-zero output for _all_ the levels in a sub-group means the OPP is + supported by hardware. A value of 0xFFFFFFFF for each level in the sub-group + will enable the OPP for all versions for the hardware. - status: Marks the node enabled/disabled. @@ -503,7 +505,6 @@ Example 5: opp-supported-hw */ opp-supported-hw = <0xF 0xFFFFFFFF 0xFFFFFFFF> opp-hz = /bits/ 64 <600000000>; - opp-microvolt = <915000 900000 925000>; ... }; @@ -516,7 +517,17 @@ Example 5: opp-supported-hw */ opp-supported-hw = <0x20 0xff0000ff 0x0000f4f0> opp-hz = /bits/ 64 <800000000>; - opp-microvolt = <915000 900000 925000>; + ... + }; + + opp-900000000 { + /* + * Supports: + * - All cuts and substrate where process version is 0x2. + * - All cuts and process where substrate version is 0x2. + */ + opp-supported-hw = <0xFFFFFFFF 0xFFFFFFFF 0x02>, <0xFFFFFFFF 0x01 0xFFFFFFFF> + opp-hz = /bits/ 64 <900000000>; ... }; }; From 0ff25c99042a56cd1580b381dd747a56286489cd Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 26 Aug 2020 16:32:27 +0530 Subject: [PATCH 112/265] opp: Allow opp-supported-hw to contain multiple versions The bindings allow multiple versions to be passed to "opp-supported-hw" property, either of which can result in enabling of the OPP. Update code to allow that. Tested-by: Stephan Gerhold Tested-by: Dmitry Osipenko Signed-off-by: Viresh Kumar --- drivers/opp/of.c | 51 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/drivers/opp/of.c b/drivers/opp/of.c index 2f294c57ec06..aa829a569825 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -434,9 +434,9 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_of_find_icc_paths); static bool _opp_is_supported(struct device *dev, struct opp_table *opp_table, struct device_node *np) { - unsigned int count = opp_table->supported_hw_count; - u32 version; - int ret; + unsigned int levels = opp_table->supported_hw_count; + int count, versions, ret, i, j; + u32 val; if (!opp_table->supported_hw) { /* @@ -451,21 +451,40 @@ static bool _opp_is_supported(struct device *dev, struct opp_table *opp_table, return true; } - while (count--) { - ret = of_property_read_u32_index(np, "opp-supported-hw", count, - &version); - if (ret) { - dev_warn(dev, "%s: failed to read opp-supported-hw property at index %d: %d\n", - __func__, count, ret); - return false; - } - - /* Both of these are bitwise masks of the versions */ - if (!(version & opp_table->supported_hw[count])) - return false; + count = of_property_count_u32_elems(np, "opp-supported-hw"); + if (count <= 0 || count % levels) { + dev_err(dev, "%s: Invalid opp-supported-hw property (%d)\n", + __func__, count); + return false; } - return true; + versions = count / levels; + + /* All levels in at least one of the versions should match */ + for (i = 0; i < versions; i++) { + bool supported = true; + + for (j = 0; j < levels; j++) { + ret = of_property_read_u32_index(np, "opp-supported-hw", + i * levels + j, &val); + if (ret) { + dev_warn(dev, "%s: failed to read opp-supported-hw property at index %d: %d\n", + __func__, i * levels + j, ret); + return false; + } + + /* Check if the level is supported */ + if (!(val & opp_table->supported_hw[j])) { + supported = false; + break; + } + } + + if (supported) + return true; + } + + return false; } static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev, From 7162fc2e4edae990e04ea7b0ad7f0106d97c9b41 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 26 Aug 2020 17:14:08 +0530 Subject: [PATCH 113/265] ARM: tegra: Pass multiple versions in opp-supported-hw property We can now pass multiple versions in "opp-supported-hw" property, lets do that and simplify the tables a bit. Reviewed-by: Dmitry Osipenko Tested-by: Dmitry Osipenko Signed-off-by: Viresh Kumar --- .../boot/dts/tegra20-cpu-opp-microvolt.dtsi | 36 - arch/arm/boot/dts/tegra20-cpu-opp.dtsi | 67 +- .../boot/dts/tegra30-cpu-opp-microvolt.dtsi | 512 ----------- arch/arm/boot/dts/tegra30-cpu-opp.dtsi | 866 ++---------------- 4 files changed, 88 insertions(+), 1393 deletions(-) diff --git a/arch/arm/boot/dts/tegra20-cpu-opp-microvolt.dtsi b/arch/arm/boot/dts/tegra20-cpu-opp-microvolt.dtsi index dce85d39480d..6f3e8c5fc5f0 100644 --- a/arch/arm/boot/dts/tegra20-cpu-opp-microvolt.dtsi +++ b/arch/arm/boot/dts/tegra20-cpu-opp-microvolt.dtsi @@ -26,14 +26,6 @@ opp-microvolt = <800000 800000 1125000>; }; - opp@456000000,800,2,2 { - opp-microvolt = <800000 800000 1125000>; - }; - - opp@456000000,800,3,2 { - opp-microvolt = <800000 800000 1125000>; - }; - opp@456000000,825 { opp-microvolt = <825000 825000 1125000>; }; @@ -46,10 +38,6 @@ opp-microvolt = <800000 800000 1125000>; }; - opp@608000000,800,3,2 { - opp-microvolt = <800000 800000 1125000>; - }; - opp@608000000,825 { opp-microvolt = <825000 825000 1125000>; }; @@ -78,18 +66,6 @@ opp-microvolt = <875000 875000 1125000>; }; - opp@760000000,875,1,1 { - opp-microvolt = <875000 875000 1125000>; - }; - - opp@760000000,875,0,2 { - opp-microvolt = <875000 875000 1125000>; - }; - - opp@760000000,875,1,2 { - opp-microvolt = <875000 875000 1125000>; - }; - opp@760000000,900 { opp-microvolt = <900000 900000 1125000>; }; @@ -134,14 +110,6 @@ opp-microvolt = <950000 950000 1125000>; }; - opp@912000000,950,0,2 { - opp-microvolt = <950000 950000 1125000>; - }; - - opp@912000000,950,2,2 { - opp-microvolt = <950000 950000 1125000>; - }; - opp@912000000,1000 { opp-microvolt = <1000000 1000000 1125000>; }; @@ -170,10 +138,6 @@ opp-microvolt = <1000000 1000000 1125000>; }; - opp@1000000000,1000,0,2 { - opp-microvolt = <1000000 1000000 1125000>; - }; - opp@1000000000,1025 { opp-microvolt = <1025000 1025000 1125000>; }; diff --git a/arch/arm/boot/dts/tegra20-cpu-opp.dtsi b/arch/arm/boot/dts/tegra20-cpu-opp.dtsi index 9b8fedb57a1b..702a635e88e7 100644 --- a/arch/arm/boot/dts/tegra20-cpu-opp.dtsi +++ b/arch/arm/boot/dts/tegra20-cpu-opp.dtsi @@ -37,19 +37,8 @@ opp@456000000,800 { clock-latency-ns = <400000>; - opp-supported-hw = <0x03 0x0006>; - opp-hz = /bits/ 64 <456000000>; - }; - - opp@456000000,800,2,2 { - clock-latency-ns = <400000>; - opp-supported-hw = <0x04 0x0004>; - opp-hz = /bits/ 64 <456000000>; - }; - - opp@456000000,800,3,2 { - clock-latency-ns = <400000>; - opp-supported-hw = <0x08 0x0004>; + opp-supported-hw = <0x03 0x0006>, <0x04 0x0004>, + <0x08 0x0004>; opp-hz = /bits/ 64 <456000000>; }; @@ -67,13 +56,7 @@ opp@608000000,800 { clock-latency-ns = <400000>; - opp-supported-hw = <0x04 0x0006>; - opp-hz = /bits/ 64 <608000000>; - }; - - opp@608000000,800,3,2 { - clock-latency-ns = <400000>; - opp-supported-hw = <0x08 0x0004>; + opp-supported-hw = <0x04 0x0006>, <0x08 0x0004>; opp-hz = /bits/ 64 <608000000>; }; @@ -115,25 +98,8 @@ opp@760000000,875 { clock-latency-ns = <400000>; - opp-supported-hw = <0x04 0x0001>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,875,1,1 { - clock-latency-ns = <400000>; - opp-supported-hw = <0x02 0x0002>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,875,0,2 { - clock-latency-ns = <400000>; - opp-supported-hw = <0x01 0x0004>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,875,1,2 { - clock-latency-ns = <400000>; - opp-supported-hw = <0x02 0x0004>; + opp-supported-hw = <0x04 0x0001>, <0x02 0x0002>, + <0x01 0x0004>, <0x02 0x0004>; opp-hz = /bits/ 64 <760000000>; }; @@ -199,19 +165,8 @@ opp@912000000,950 { clock-latency-ns = <400000>; - opp-supported-hw = <0x02 0x0006>; - opp-hz = /bits/ 64 <912000000>; - }; - - opp@912000000,950,0,2 { - clock-latency-ns = <400000>; - opp-supported-hw = <0x01 0x0004>; - opp-hz = /bits/ 64 <912000000>; - }; - - opp@912000000,950,2,2 { - clock-latency-ns = <400000>; - opp-supported-hw = <0x04 0x0004>; + opp-supported-hw = <0x02 0x0006>, <0x01 0x0004>, + <0x04 0x0004>; opp-hz = /bits/ 64 <912000000>; }; @@ -253,13 +208,7 @@ opp@1000000000,1000 { clock-latency-ns = <400000>; - opp-supported-hw = <0x02 0x0006>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,1000,0,2 { - clock-latency-ns = <400000>; - opp-supported-hw = <0x01 0x0004>; + opp-supported-hw = <0x02 0x0006>, <0x01 0x0004>; opp-hz = /bits/ 64 <1000000000>; }; diff --git a/arch/arm/boot/dts/tegra30-cpu-opp-microvolt.dtsi b/arch/arm/boot/dts/tegra30-cpu-opp-microvolt.dtsi index d682f7437146..1be715d2a442 100644 --- a/arch/arm/boot/dts/tegra30-cpu-opp-microvolt.dtsi +++ b/arch/arm/boot/dts/tegra30-cpu-opp-microvolt.dtsi @@ -74,22 +74,6 @@ opp-microvolt = <850000 850000 1250000>; }; - opp@475000000,850,0,1 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@475000000,850,0,4 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@475000000,850,0,7 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@475000000,850,0,8 { - opp-microvolt = <850000 850000 1250000>; - }; - opp@608000000,850 { opp-microvolt = <850000 850000 1250000>; }; @@ -106,62 +90,6 @@ opp-microvolt = <850000 850000 1250000>; }; - opp@640000000,850,1,1 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,2,1 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,3,1 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,1,4 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,2,4 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,3,4 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,1,7 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,2,7 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,3,7 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,4,7 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,1,8 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,2,8 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,3,8 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@640000000,850,4,8 { - opp-microvolt = <850000 850000 1250000>; - }; - opp@640000000,900 { opp-microvolt = <900000 900000 1250000>; }; @@ -170,94 +98,10 @@ opp-microvolt = <850000 850000 1250000>; }; - opp@760000000,850,3,1 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@760000000,850,3,2 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@760000000,850,3,3 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@760000000,850,3,4 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@760000000,850,3,7 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@760000000,850,4,7 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@760000000,850,3,8 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@760000000,850,4,8 { - opp-microvolt = <850000 850000 1250000>; - }; - - opp@760000000,850,0,10 { - opp-microvolt = <850000 850000 1250000>; - }; - opp@760000000,900 { opp-microvolt = <900000 900000 1250000>; }; - opp@760000000,900,1,1 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@760000000,900,2,1 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@760000000,900,1,2 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@760000000,900,2,2 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@760000000,900,1,3 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@760000000,900,2,3 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@760000000,900,1,4 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@760000000,900,2,4 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@760000000,900,1,7 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@760000000,900,2,7 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@760000000,900,1,8 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@760000000,900,2,8 { - opp-microvolt = <900000 900000 1250000>; - }; - opp@760000000,912 { opp-microvolt = <912000 912000 1250000>; }; @@ -282,90 +126,10 @@ opp-microvolt = <900000 900000 1250000>; }; - opp@860000000,900,2,1 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,3,1 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,2,2 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,3,2 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,2,3 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,3,3 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,2,4 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,3,4 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,2,7 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,3,7 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,4,7 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,2,8 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,3,8 { - opp-microvolt = <900000 900000 1250000>; - }; - - opp@860000000,900,4,8 { - opp-microvolt = <900000 900000 1250000>; - }; - opp@860000000,975 { opp-microvolt = <975000 975000 1250000>; }; - opp@860000000,975,1,1 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@860000000,975,1,2 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@860000000,975,1,3 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@860000000,975,1,4 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@860000000,975,1,7 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@860000000,975,1,8 { - opp-microvolt = <975000 975000 1250000>; - }; - opp@860000000,1000 { opp-microvolt = <1000000 1000000 1250000>; }; @@ -382,62 +146,6 @@ opp-microvolt = <975000 975000 1250000>; }; - opp@1000000000,975,2,1 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,3,1 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,2,2 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,3,2 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,2,3 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,3,3 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,2,4 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,3,4 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,2,7 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,3,7 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,4,7 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,2,8 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,3,8 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1000000000,975,4,8 { - opp-microvolt = <975000 975000 1250000>; - }; - opp@1000000000,1000 { opp-microvolt = <1000000 1000000 1250000>; }; @@ -454,66 +162,10 @@ opp-microvolt = <975000 975000 1250000>; }; - opp@1100000000,975,3,1 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1100000000,975,3,2 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1100000000,975,3,3 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1100000000,975,3,4 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1100000000,975,3,7 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1100000000,975,4,7 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1100000000,975,3,8 { - opp-microvolt = <975000 975000 1250000>; - }; - - opp@1100000000,975,4,8 { - opp-microvolt = <975000 975000 1250000>; - }; - opp@1100000000,1000 { opp-microvolt = <1000000 1000000 1250000>; }; - opp@1100000000,1000,2,1 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1100000000,1000,2,2 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1100000000,1000,2,3 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1100000000,1000,2,4 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1100000000,1000,2,7 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1100000000,1000,2,8 { - opp-microvolt = <1000000 1000000 1250000>; - }; - opp@1100000000,1025 { opp-microvolt = <1025000 1025000 1250000>; }; @@ -534,66 +186,10 @@ opp-microvolt = <1000000 1000000 1250000>; }; - opp@1200000000,1000,3,1 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1200000000,1000,3,2 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1200000000,1000,3,3 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1200000000,1000,3,4 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1200000000,1000,3,7 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1200000000,1000,4,7 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1200000000,1000,3,8 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1200000000,1000,4,8 { - opp-microvolt = <1000000 1000000 1250000>; - }; - opp@1200000000,1025 { opp-microvolt = <1025000 1025000 1250000>; }; - opp@1200000000,1025,2,1 { - opp-microvolt = <1025000 1025000 1250000>; - }; - - opp@1200000000,1025,2,2 { - opp-microvolt = <1025000 1025000 1250000>; - }; - - opp@1200000000,1025,2,3 { - opp-microvolt = <1025000 1025000 1250000>; - }; - - opp@1200000000,1025,2,4 { - opp-microvolt = <1025000 1025000 1250000>; - }; - - opp@1200000000,1025,2,7 { - opp-microvolt = <1025000 1025000 1250000>; - }; - - opp@1200000000,1025,2,8 { - opp-microvolt = <1025000 1025000 1250000>; - }; - opp@1200000000,1050 { opp-microvolt = <1050000 1050000 1250000>; }; @@ -610,90 +206,18 @@ opp-microvolt = <1000000 1000000 1250000>; }; - opp@1300000000,1000,4,7 { - opp-microvolt = <1000000 1000000 1250000>; - }; - - opp@1300000000,1000,4,8 { - opp-microvolt = <1000000 1000000 1250000>; - }; - opp@1300000000,1025 { opp-microvolt = <1025000 1025000 1250000>; }; - opp@1300000000,1025,3,1 { - opp-microvolt = <1025000 1025000 1250000>; - }; - - opp@1300000000,1025,3,7 { - opp-microvolt = <1025000 1025000 1250000>; - }; - - opp@1300000000,1025,3,8 { - opp-microvolt = <1025000 1025000 1250000>; - }; - opp@1300000000,1050 { opp-microvolt = <1050000 1050000 1250000>; }; - opp@1300000000,1050,2,1 { - opp-microvolt = <1050000 1050000 1250000>; - }; - - opp@1300000000,1050,3,2 { - opp-microvolt = <1050000 1050000 1250000>; - }; - - opp@1300000000,1050,3,3 { - opp-microvolt = <1050000 1050000 1250000>; - }; - - opp@1300000000,1050,3,4 { - opp-microvolt = <1050000 1050000 1250000>; - }; - - opp@1300000000,1050,3,5 { - opp-microvolt = <1050000 1050000 1250000>; - }; - - opp@1300000000,1050,3,6 { - opp-microvolt = <1050000 1050000 1250000>; - }; - - opp@1300000000,1050,2,7 { - opp-microvolt = <1050000 1050000 1250000>; - }; - - opp@1300000000,1050,2,8 { - opp-microvolt = <1050000 1050000 1250000>; - }; - - opp@1300000000,1050,3,12 { - opp-microvolt = <1050000 1050000 1250000>; - }; - - opp@1300000000,1050,3,13 { - opp-microvolt = <1050000 1050000 1250000>; - }; - opp@1300000000,1075 { opp-microvolt = <1075000 1075000 1250000>; }; - opp@1300000000,1075,2,2 { - opp-microvolt = <1075000 1075000 1250000>; - }; - - opp@1300000000,1075,2,3 { - opp-microvolt = <1075000 1075000 1250000>; - }; - - opp@1300000000,1075,2,4 { - opp-microvolt = <1075000 1075000 1250000>; - }; - opp@1300000000,1100 { opp-microvolt = <1100000 1100000 1250000>; }; @@ -722,10 +246,6 @@ opp-microvolt = <1150000 1150000 1250000>; }; - opp@1400000000,1150,2,4 { - opp-microvolt = <1150000 1150000 1250000>; - }; - opp@1400000000,1175 { opp-microvolt = <1175000 1175000 1250000>; }; @@ -738,42 +258,10 @@ opp-microvolt = <1125000 1125000 1250000>; }; - opp@1500000000,1125,4,5 { - opp-microvolt = <1125000 1125000 1250000>; - }; - - opp@1500000000,1125,4,6 { - opp-microvolt = <1125000 1125000 1250000>; - }; - - opp@1500000000,1125,4,12 { - opp-microvolt = <1125000 1125000 1250000>; - }; - - opp@1500000000,1125,4,13 { - opp-microvolt = <1125000 1125000 1250000>; - }; - opp@1500000000,1150 { opp-microvolt = <1150000 1150000 1250000>; }; - opp@1500000000,1150,3,5 { - opp-microvolt = <1150000 1150000 1250000>; - }; - - opp@1500000000,1150,3,6 { - opp-microvolt = <1150000 1150000 1250000>; - }; - - opp@1500000000,1150,3,12 { - opp-microvolt = <1150000 1150000 1250000>; - }; - - opp@1500000000,1150,3,13 { - opp-microvolt = <1150000 1150000 1250000>; - }; - opp@1500000000,1200 { opp-microvolt = <1200000 1200000 1250000>; }; diff --git a/arch/arm/boot/dts/tegra30-cpu-opp.dtsi b/arch/arm/boot/dts/tegra30-cpu-opp.dtsi index 8e434f6713cd..0f7135006d19 100644 --- a/arch/arm/boot/dts/tegra30-cpu-opp.dtsi +++ b/arch/arm/boot/dts/tegra30-cpu-opp.dtsi @@ -109,31 +109,9 @@ opp@475000000,850 { clock-latency-ns = <100000>; - opp-supported-hw = <0x0F 0x0001>; - opp-hz = /bits/ 64 <475000000>; - }; - - opp@475000000,850,0,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x01 0x0002>; - opp-hz = /bits/ 64 <475000000>; - }; - - opp@475000000,850,0,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x01 0x0010>; - opp-hz = /bits/ 64 <475000000>; - }; - - opp@475000000,850,0,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x01 0x0080>; - opp-hz = /bits/ 64 <475000000>; - }; - - opp@475000000,850,0,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x01 0x0100>; + opp-supported-hw = <0x0F 0x0001>, <0x01 0x0002>, + <0x01 0x0010>, <0x01 0x0080>, + <0x01 0x0100>; opp-hz = /bits/ 64 <475000000>; }; @@ -157,91 +135,14 @@ opp@640000000,850 { clock-latency-ns = <100000>; - opp-supported-hw = <0x0F 0x0001>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,1,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0002>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,2,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0002>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,3,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0002>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,1,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0010>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,2,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0010>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,3,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0010>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,1,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0080>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,2,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0080>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,3,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0080>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,4,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0080>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,1,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0100>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,2,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0100>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,3,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0100>; - opp-hz = /bits/ 64 <640000000>; - }; - - opp@640000000,850,4,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0100>; + opp-supported-hw = <0x0F 0x0001>, <0x02 0x0002>, + <0x04 0x0002>, <0x08 0x0002>, + <0x02 0x0010>, <0x04 0x0010>, + <0x08 0x0010>, <0x02 0x0080>, + <0x04 0x0080>, <0x08 0x0080>, + <0x10 0x0080>, <0x02 0x0100>, + <0x04 0x0100>, <0x08 0x0100>, + <0x10 0x0100>; opp-hz = /bits/ 64 <640000000>; }; @@ -253,139 +154,23 @@ opp@760000000,850 { clock-latency-ns = <100000>; - opp-supported-hw = <0x1E 0x3461>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,850,3,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0002>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,850,3,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0004>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,850,3,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0008>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,850,3,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0010>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,850,3,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0080>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,850,4,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0080>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,850,3,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0100>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,850,4,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0100>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,850,0,10 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x01 0x0400>; + opp-supported-hw = <0x1E 0x3461>, <0x08 0x0002>, + <0x08 0x0004>, <0x08 0x0008>, + <0x08 0x0010>, <0x08 0x0080>, + <0x10 0x0080>, <0x08 0x0100>, + <0x10 0x0100>, <0x01 0x0400>; opp-hz = /bits/ 64 <760000000>; }; opp@760000000,900 { clock-latency-ns = <100000>; - opp-supported-hw = <0x01 0x0001>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,900,1,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0002>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,900,2,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0002>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,900,1,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0004>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,900,2,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0004>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,900,1,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0008>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,900,2,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0008>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,900,1,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0010>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,900,2,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0010>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,900,1,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0080>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,900,2,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0080>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,900,1,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0100>; - opp-hz = /bits/ 64 <760000000>; - }; - - opp@760000000,900,2,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0100>; + opp-supported-hw = <0x01 0x0001>, <0x02 0x0002>, + <0x04 0x0002>, <0x02 0x0004>, + <0x04 0x0004>, <0x02 0x0008>, + <0x04 0x0008>, <0x02 0x0010>, + <0x04 0x0010>, <0x02 0x0080>, + <0x04 0x0080>, <0x02 0x0100>, + <0x04 0x0100>; opp-hz = /bits/ 64 <760000000>; }; @@ -421,133 +206,23 @@ opp@860000000,900 { clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0001>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,2,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0002>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,3,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0002>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,2,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0004>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,3,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0004>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,2,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0008>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,3,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0008>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,2,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0010>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,3,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0010>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,2,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0080>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,3,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0080>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,4,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0080>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,2,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0100>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,3,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0100>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,900,4,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0100>; + opp-supported-hw = <0x02 0x0001>, <0x04 0x0002>, + <0x08 0x0002>, <0x04 0x0004>, + <0x08 0x0004>, <0x04 0x0008>, + <0x08 0x0008>, <0x04 0x0010>, + <0x08 0x0010>, <0x04 0x0080>, + <0x08 0x0080>, <0x10 0x0080>, + <0x04 0x0100>, <0x08 0x0100>, + <0x10 0x0100>; opp-hz = /bits/ 64 <860000000>; }; opp@860000000,975 { clock-latency-ns = <100000>; - opp-supported-hw = <0x01 0x0001>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,975,1,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0002>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,975,1,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0004>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,975,1,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0008>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,975,1,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0010>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,975,1,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0080>; - opp-hz = /bits/ 64 <860000000>; - }; - - opp@860000000,975,1,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0100>; + opp-supported-hw = <0x01 0x0001>, <0x02 0x0002>, + <0x02 0x0004>, <0x02 0x0008>, + <0x02 0x0010>, <0x02 0x0080>, + <0x02 0x0100>; opp-hz = /bits/ 64 <860000000>; }; @@ -571,91 +246,14 @@ opp@1000000000,975 { clock-latency-ns = <100000>; - opp-supported-hw = <0x03 0x0001>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,2,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0002>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,3,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0002>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,2,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0004>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,3,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0004>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,2,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0008>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,3,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0008>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,2,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0010>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,3,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0010>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,2,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0080>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,3,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0080>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,4,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0080>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,2,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0100>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,3,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0100>; - opp-hz = /bits/ 64 <1000000000>; - }; - - opp@1000000000,975,4,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0100>; + opp-supported-hw = <0x03 0x0001>, <0x04 0x0002>, + <0x08 0x0002>, <0x04 0x0004>, + <0x08 0x0004>, <0x04 0x0008>, + <0x08 0x0008>, <0x04 0x0010>, + <0x08 0x0010>, <0x04 0x0080>, + <0x08 0x0080>, <0x10 0x0080>, + <0x04 0x0100>, <0x08 0x0100>, + <0x10 0x0100>; opp-hz = /bits/ 64 <1000000000>; }; @@ -679,97 +277,20 @@ opp@1100000000,975 { clock-latency-ns = <100000>; - opp-supported-hw = <0x06 0x0001>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,975,3,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0002>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,975,3,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0004>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,975,3,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0008>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,975,3,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0010>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,975,3,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0080>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,975,4,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0080>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,975,3,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0100>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,975,4,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0100>; + opp-supported-hw = <0x06 0x0001>, <0x08 0x0002>, + <0x08 0x0004>, <0x08 0x0008>, + <0x08 0x0010>, <0x08 0x0080>, + <0x10 0x0080>, <0x08 0x0100>, + <0x10 0x0100>; opp-hz = /bits/ 64 <1100000000>; }; opp@1100000000,1000 { clock-latency-ns = <100000>; - opp-supported-hw = <0x01 0x0001>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,1000,2,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0002>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,1000,2,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0004>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,1000,2,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0008>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,1000,2,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0010>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,1000,2,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0080>; - opp-hz = /bits/ 64 <1100000000>; - }; - - opp@1100000000,1000,2,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0100>; + opp-supported-hw = <0x01 0x0001>, <0x04 0x0002>, + <0x04 0x0004>, <0x04 0x0008>, + <0x04 0x0010>, <0x04 0x0080>, + <0x04 0x0100>; opp-hz = /bits/ 64 <1100000000>; }; @@ -799,97 +320,20 @@ opp@1200000000,1000 { clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0001>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1000,3,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0002>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1000,3,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0004>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1000,3,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0008>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1000,3,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0010>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1000,3,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0080>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1000,4,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0080>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1000,3,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0100>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1000,4,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0100>; + opp-supported-hw = <0x04 0x0001>, <0x08 0x0002>, + <0x08 0x0004>, <0x08 0x0008>, + <0x08 0x0010>, <0x08 0x0080>, + <0x10 0x0080>, <0x08 0x0100>, + <0x10 0x0100>; opp-hz = /bits/ 64 <1200000000>; }; opp@1200000000,1025 { clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0001>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1025,2,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0002>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1025,2,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0004>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1025,2,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0008>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1025,2,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0010>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1025,2,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0080>; - opp-hz = /bits/ 64 <1200000000>; - }; - - opp@1200000000,1025,2,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0100>; + opp-supported-hw = <0x02 0x0001>, <0x04 0x0002>, + <0x04 0x0004>, <0x04 0x0008>, + <0x04 0x0010>, <0x04 0x0080>, + <0x04 0x0100>; opp-hz = /bits/ 64 <1200000000>; }; @@ -913,133 +357,33 @@ opp@1300000000,1000 { clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0001>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1000,4,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0080>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1000,4,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0100>; + opp-supported-hw = <0x08 0x0001>, <0x10 0x0080>, + <0x10 0x0100>; opp-hz = /bits/ 64 <1300000000>; }; opp@1300000000,1025 { clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0001>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1025,3,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0002>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1025,3,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0080>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1025,3,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0100>; + opp-supported-hw = <0x04 0x0001>, <0x08 0x0002>, + <0x08 0x0080>, <0x08 0x0100>; opp-hz = /bits/ 64 <1300000000>; }; opp@1300000000,1050 { clock-latency-ns = <100000>; - opp-supported-hw = <0x12 0x3061>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1050,2,1 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0002>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1050,3,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0004>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1050,3,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0008>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1050,3,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0010>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1050,3,5 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0020>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1050,3,6 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0040>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1050,2,7 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0080>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1050,2,8 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0100>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1050,3,12 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x1000>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1050,3,13 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x2000>; + opp-supported-hw = <0x12 0x3061>, <0x04 0x0002>, + <0x08 0x0004>, <0x08 0x0008>, + <0x08 0x0010>, <0x08 0x0020>, + <0x08 0x0040>, <0x04 0x0080>, + <0x04 0x0100>, <0x08 0x1000>, + <0x08 0x2000>; opp-hz = /bits/ 64 <1300000000>; }; opp@1300000000,1075 { clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x0182>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1075,2,2 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0004>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1075,2,3 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0008>; - opp-hz = /bits/ 64 <1300000000>; - }; - - opp@1300000000,1075,2,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0010>; + opp-supported-hw = <0x02 0x0182>, <0x04 0x0004>, + <0x04 0x0008>, <0x04 0x0010>; opp-hz = /bits/ 64 <1300000000>; }; @@ -1081,13 +425,7 @@ opp@1400000000,1150 { clock-latency-ns = <100000>; - opp-supported-hw = <0x02 0x000C>; - opp-hz = /bits/ 64 <1400000000>; - }; - - opp@1400000000,1150,2,4 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0010>; + opp-supported-hw = <0x02 0x000C>, <0x04 0x0010>; opp-hz = /bits/ 64 <1400000000>; }; @@ -1105,61 +443,17 @@ opp@1500000000,1125 { clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0010>; - opp-hz = /bits/ 64 <1500000000>; - }; - - opp@1500000000,1125,4,5 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0020>; - opp-hz = /bits/ 64 <1500000000>; - }; - - opp@1500000000,1125,4,6 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x0040>; - opp-hz = /bits/ 64 <1500000000>; - }; - - opp@1500000000,1125,4,12 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x1000>; - opp-hz = /bits/ 64 <1500000000>; - }; - - opp@1500000000,1125,4,13 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x10 0x2000>; + opp-supported-hw = <0x08 0x0010>, <0x10 0x0020>, + <0x10 0x0040>, <0x10 0x1000>, + <0x10 0x2000>; opp-hz = /bits/ 64 <1500000000>; }; opp@1500000000,1150 { clock-latency-ns = <100000>; - opp-supported-hw = <0x04 0x0010>; - opp-hz = /bits/ 64 <1500000000>; - }; - - opp@1500000000,1150,3,5 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0020>; - opp-hz = /bits/ 64 <1500000000>; - }; - - opp@1500000000,1150,3,6 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x0040>; - opp-hz = /bits/ 64 <1500000000>; - }; - - opp@1500000000,1150,3,12 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x1000>; - opp-hz = /bits/ 64 <1500000000>; - }; - - opp@1500000000,1150,3,13 { - clock-latency-ns = <100000>; - opp-supported-hw = <0x08 0x2000>; + opp-supported-hw = <0x04 0x0010>, <0x08 0x0020>, + <0x08 0x0040>, <0x08 0x1000>, + <0x08 0x2000>; opp-hz = /bits/ 64 <1500000000>; }; From cb60e9602cce1593eb1e9cdc8ee562815078a354 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 31 Aug 2020 11:22:37 +0530 Subject: [PATCH 114/265] opp: Prevent memory leak in dev_pm_opp_attach_genpd() If dev_pm_opp_attach_genpd() is called multiple times (once for each CPU sharing the table), then it would result in unwanted behavior like memory leak, attaching the domain multiple times, etc. Handle that by checking and returning earlier if the domains are already attached. Now that dev_pm_opp_detach_genpd() can get called multiple times as well, we need to protect that too. Note that the virtual device pointers aren't returned in this case, as they may become unavailable to some callers during the middle of the operation. Reported-by: Stephan Gerhold Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 000d0fcb4680..e65174725a4d 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -1960,6 +1960,9 @@ static void _opp_detach_genpd(struct opp_table *opp_table) { int index; + if (!opp_table->genpd_virt_devs) + return; + for (index = 0; index < opp_table->required_opp_count; index++) { if (!opp_table->genpd_virt_devs[index]) continue; @@ -2006,6 +2009,9 @@ struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, if (IS_ERR(opp_table)) return opp_table; + if (opp_table->genpd_virt_devs) + return opp_table; + /* * If the genpd's OPP table isn't already initialized, parsing of the * required-opps fail for dev. We should retry this after genpd's OPP From a5663c9b1e31c00e0bdfaf4d92eb51358fc3950f Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Fri, 11 Sep 2020 14:56:18 +0530 Subject: [PATCH 115/265] opp: Allow opp-level to be set to 0 The DT bindings don't put such a constraint, nor should the kernel. It is perfectly fine for opp-level to be set to 0, if we need to put the performance state votes for a domain for a particular OPP. Reported-by: Stephan Gerhold Tested-by: Stephan Gerhold Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 3 --- drivers/opp/of.c | 20 +++++++------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/drivers/opp/core.c b/drivers/opp/core.c index e65174725a4d..8d047300d001 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2109,9 +2109,6 @@ int dev_pm_opp_xlate_performance_state(struct opp_table *src_table, int dest_pstate = -EINVAL; int i; - if (!pstate) - return 0; - /* * Normally the src_table will have the "required_opps" property set to * point to one of the OPPs in the dst_table, but in some cases the diff --git a/drivers/opp/of.c b/drivers/opp/of.c index aa829a569825..874b58756220 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -842,7 +842,7 @@ free_opp: static int _of_add_opp_table_v2(struct device *dev, struct opp_table *opp_table) { struct device_node *np; - int ret, count = 0, pstate_count = 0; + int ret, count = 0; struct dev_pm_opp *opp; /* OPP table is already initialized for the device */ @@ -876,20 +876,14 @@ static int _of_add_opp_table_v2(struct device *dev, struct opp_table *opp_table) goto remove_static_opp; } - list_for_each_entry(opp, &opp_table->opp_list, node) - pstate_count += !!opp->pstate; - - /* Either all or none of the nodes shall have performance state set */ - if (pstate_count && pstate_count != count) { - dev_err(dev, "Not all nodes have performance state set (%d: %d)\n", - count, pstate_count); - ret = -ENOENT; - goto remove_static_opp; + list_for_each_entry(opp, &opp_table->opp_list, node) { + /* Any non-zero performance state would enable the feature */ + if (opp->pstate) { + opp_table->genpd_performance_state = true; + break; + } } - if (pstate_count) - opp_table->genpd_performance_state = true; - return 0; remove_static_opp: From b89c01c960511dcffe3666d89645c95445d00902 Mon Sep 17 00:00:00 2001 From: Jon Hunter Date: Mon, 24 Aug 2020 15:59:07 +0100 Subject: [PATCH 116/265] cpufreq: tegra186: Fix initial frequency Commit 6cc3d0e9a097 ("cpufreq: tegra186: add CPUFREQ_NEED_INITIAL_FREQ_CHECK flag") fixed CPUFREQ support for Tegra186 but as a consequence the following warnings are now seen on boot ... cpufreq: cpufreq_online: CPU0: Running at unlisted freq: 0 KHz cpufreq: cpufreq_online: CPU0: Unlisted initial frequency changed to: 2035200 KHz cpufreq: cpufreq_online: CPU1: Running at unlisted freq: 0 KHz cpufreq: cpufreq_online: CPU1: Unlisted initial frequency changed to: 2035200 KHz cpufreq: cpufreq_online: CPU2: Running at unlisted freq: 0 KHz cpufreq: cpufreq_online: CPU2: Unlisted initial frequency changed to: 2035200 KHz cpufreq: cpufreq_online: CPU3: Running at unlisted freq: 0 KHz cpufreq: cpufreq_online: CPU3: Unlisted initial frequency changed to: 2035200 KHz cpufreq: cpufreq_online: CPU4: Running at unlisted freq: 0 KHz cpufreq: cpufreq_online: CPU4: Unlisted initial frequency changed to: 2035200 KHz cpufreq: cpufreq_online: CPU5: Running at unlisted freq: 0 KHz cpufreq: cpufreq_online: CPU5: Unlisted initial frequency changed to: 2035200 KHz Fix this by adding a 'get' callback for the Tegra186 CPUFREQ driver to retrieve the current operating frequency for a given CPU. The 'get' callback uses the current 'ndiv' value that is programmed to determine that current operating frequency. Signed-off-by: Jon Hunter [ Viresh: Return 0 on error ] Signed-off-by: Viresh Kumar --- drivers/cpufreq/tegra186-cpufreq.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/drivers/cpufreq/tegra186-cpufreq.c b/drivers/cpufreq/tegra186-cpufreq.c index 01e1f58ba422..4b4079f51559 100644 --- a/drivers/cpufreq/tegra186-cpufreq.c +++ b/drivers/cpufreq/tegra186-cpufreq.c @@ -14,6 +14,7 @@ #define EDVD_CORE_VOLT_FREQ(core) (0x20 + (core) * 0x4) #define EDVD_CORE_VOLT_FREQ_F_SHIFT 0 +#define EDVD_CORE_VOLT_FREQ_F_MASK 0xffff #define EDVD_CORE_VOLT_FREQ_V_SHIFT 16 struct tegra186_cpufreq_cluster_info { @@ -91,10 +92,39 @@ static int tegra186_cpufreq_set_target(struct cpufreq_policy *policy, return 0; } +static unsigned int tegra186_cpufreq_get(unsigned int cpu) +{ + struct cpufreq_frequency_table *tbl; + struct cpufreq_policy *policy; + void __iomem *edvd_reg; + unsigned int i, freq = 0; + u32 ndiv; + + policy = cpufreq_cpu_get(cpu); + if (!policy) + return 0; + + tbl = policy->freq_table; + edvd_reg = policy->driver_data; + ndiv = readl(edvd_reg) & EDVD_CORE_VOLT_FREQ_F_MASK; + + for (i = 0; tbl[i].frequency != CPUFREQ_TABLE_END; i++) { + if ((tbl[i].driver_data & EDVD_CORE_VOLT_FREQ_F_MASK) == ndiv) { + freq = tbl[i].frequency; + break; + } + } + + cpufreq_cpu_put(policy); + + return freq; +} + static struct cpufreq_driver tegra186_cpufreq_driver = { .name = "tegra186", .flags = CPUFREQ_STICKY | CPUFREQ_HAVE_GOVERNOR_PER_POLICY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .get = tegra186_cpufreq_get, .verify = cpufreq_generic_frequency_table_verify, .target_index = tegra186_cpufreq_set_target, .init = tegra186_cpufreq_init, From 629238068eb905297d5edc710c41076456ae8338 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Wed, 26 Aug 2020 18:00:15 +0200 Subject: [PATCH 117/265] cpufreq: s5pv210: Simplify with dev_err_probe() Common pattern of handling deferred probe can be simplified with dev_err_probe(). Less code and also it prints the error value. Signed-off-by: Krzysztof Kozlowski Signed-off-by: Viresh Kumar --- drivers/cpufreq/s5pv210-cpufreq.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c index e84281e2561d..7dccdb364fcf 100644 --- a/drivers/cpufreq/s5pv210-cpufreq.c +++ b/drivers/cpufreq/s5pv210-cpufreq.c @@ -590,6 +590,7 @@ static struct notifier_block s5pv210_cpufreq_reboot_notifier = { static int s5pv210_cpufreq_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; struct device_node *np; int id, result = 0; @@ -602,21 +603,14 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev) * cpufreq-dt driver. */ arm_regulator = regulator_get(NULL, "vddarm"); - if (IS_ERR(arm_regulator)) { - if (PTR_ERR(arm_regulator) == -EPROBE_DEFER) - pr_debug("vddarm regulator not ready, defer\n"); - else - pr_err("failed to get regulator vddarm\n"); - return PTR_ERR(arm_regulator); - } + if (IS_ERR(arm_regulator)) + return dev_err_probe(dev, PTR_ERR(arm_regulator), + "failed to get regulator vddarm\n"); int_regulator = regulator_get(NULL, "vddint"); if (IS_ERR(int_regulator)) { - if (PTR_ERR(int_regulator) == -EPROBE_DEFER) - pr_debug("vddint regulator not ready, defer\n"); - else - pr_err("failed to get regulator vddint\n"); - result = PTR_ERR(int_regulator); + result = dev_err_probe(dev, PTR_ERR(int_regulator), + "failed to get regulator vddint\n"); goto err_int_regulator; } From 77c6d5cd93f52e6692fbe570594d4e14ec0b63e8 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Wed, 26 Aug 2020 18:00:16 +0200 Subject: [PATCH 118/265] cpufreq: s5pv210: Use dev_err instead of pr_err in probe dev_err() allows easily to identify the device printing the message so no need for __func__. Signed-off-by: Krzysztof Kozlowski [ Viresh: Don't remove update to result variable ] Signed-off-by: Viresh Kumar --- drivers/cpufreq/s5pv210-cpufreq.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c index 7dccdb364fcf..bed496cf8d24 100644 --- a/drivers/cpufreq/s5pv210-cpufreq.c +++ b/drivers/cpufreq/s5pv210-cpufreq.c @@ -616,8 +616,7 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev) np = of_find_compatible_node(NULL, NULL, "samsung,s5pv210-clock"); if (!np) { - pr_err("%s: failed to find clock controller DT node\n", - __func__); + dev_err(dev, "failed to find clock controller DT node\n"); result = -ENODEV; goto err_clock; } @@ -625,7 +624,7 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev) clk_base = of_iomap(np, 0); of_node_put(np); if (!clk_base) { - pr_err("%s: failed to map clock registers\n", __func__); + dev_err(dev, "failed to map clock registers\n"); result = -EFAULT; goto err_clock; } @@ -633,8 +632,7 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev) for_each_compatible_node(np, NULL, "samsung,s5pv210-dmc") { id = of_alias_get_id(np, "dmc"); if (id < 0 || id >= ARRAY_SIZE(dmc_base)) { - pr_err("%s: failed to get alias of dmc node '%pOFn'\n", - __func__, np); + dev_err(dev, "failed to get alias of dmc node '%pOFn'\n", np); of_node_put(np); result = id; goto err_clk_base; @@ -642,8 +640,7 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev) dmc_base[id] = of_iomap(np, 0); if (!dmc_base[id]) { - pr_err("%s: failed to map dmc%d registers\n", - __func__, id); + dev_err(dev, "failed to map dmc%d registers\n", id); of_node_put(np); result = -EFAULT; goto err_dmc; @@ -652,7 +649,7 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev) for (id = 0; id < ARRAY_SIZE(dmc_base); ++id) { if (!dmc_base[id]) { - pr_err("%s: failed to find dmc%d node\n", __func__, id); + dev_err(dev, "failed to find dmc%d node\n", id); result = -ENODEV; goto err_dmc; } From 01a163c52039e9426c7d3d3ab16ca261ad622597 Mon Sep 17 00:00:00 2001 From: Alain Volmat Date: Mon, 31 Aug 2020 08:10:11 +0200 Subject: [PATCH 119/265] cpufreq: sti-cpufreq: add stih418 support The STiH418 can be controlled the same way as STiH407 & STiH410 regarding cpufreq. Signed-off-by: Alain Volmat Signed-off-by: Viresh Kumar --- drivers/cpufreq/sti-cpufreq.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/sti-cpufreq.c b/drivers/cpufreq/sti-cpufreq.c index a5ad96d29adc..4ac6fb23792a 100644 --- a/drivers/cpufreq/sti-cpufreq.c +++ b/drivers/cpufreq/sti-cpufreq.c @@ -141,7 +141,8 @@ static const struct reg_field sti_stih407_dvfs_regfields[DVFS_MAX_REGFIELDS] = { static const struct reg_field *sti_cpufreq_match(void) { if (of_machine_is_compatible("st,stih407") || - of_machine_is_compatible("st,stih410")) + of_machine_is_compatible("st,stih410") || + of_machine_is_compatible("st,stih418")) return sti_stih407_dvfs_regfields; return NULL; @@ -258,7 +259,8 @@ static int sti_cpufreq_init(void) int ret; if ((!of_machine_is_compatible("st,stih407")) && - (!of_machine_is_compatible("st,stih410"))) + (!of_machine_is_compatible("st,stih410")) && + (!of_machine_is_compatible("st,stih418"))) return -ENODEV; ddata.cpu = get_cpu_device(0); From 305accf3b53ae679c1333022c33f95a7e337a471 Mon Sep 17 00:00:00 2001 From: Alain Volmat Date: Mon, 31 Aug 2020 08:10:12 +0200 Subject: [PATCH 120/265] cpufreq: dt-platdev: Blacklist st,stih418 SoC Add st,stih418 SoC in the blacklist since the cpufreq driver for this platform is already registering the driver. Signed-off-by: Alain Volmat Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq-dt-platdev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index 7d01df7bfa6c..3776d960f405 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -137,6 +137,7 @@ static const struct of_device_id blacklist[] __initconst = { { .compatible = "st,stih407", }, { .compatible = "st,stih410", }, + { .compatible = "st,stih418", }, { .compatible = "sigma,tango4", }, From a0d698d8c21aaa71b2a2fc6c938d658797e31911 Mon Sep 17 00:00:00 2001 From: Alain Volmat Date: Mon, 31 Aug 2020 08:10:13 +0200 Subject: [PATCH 121/265] cpufreq: arm: Kconfig: add CPUFREQ_DT depend for STI CPUFREQ The sti cpufreq driver is relying on the CPUFREQ_DT driver hence add the depends within the Kconfig.arm Signed-off-by: Alain Volmat Signed-off-by: Viresh Kumar --- drivers/cpufreq/Kconfig.arm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm index cb72fb507d57..bf5830eb664f 100644 --- a/drivers/cpufreq/Kconfig.arm +++ b/drivers/cpufreq/Kconfig.arm @@ -283,7 +283,7 @@ config ARM_SPEAR_CPUFREQ config ARM_STI_CPUFREQ tristate "STi CPUFreq support" - depends on SOC_STIH407 + depends on CPUFREQ_DT && SOC_STIH407 help This driver uses the generic OPP framework to match the running platform with a predefined set of suitable values. If not provided From c942d1542f1bc5001216fabce9cb8ffbe515777e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pali=20Roh=C3=A1r?= Date: Mon, 7 Sep 2020 15:27:16 +0200 Subject: [PATCH 122/265] cpufreq: armada-37xx: Add missing MODULE_DEVICE_TABLE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CONFIG_ARM_ARMADA_37XX_CPUFREQ is tristate option and therefore this cpufreq driver can be compiled as a module. This patch adds missing MODULE_DEVICE_TABLE which generates correct modalias for automatic loading of this cpufreq driver when is compiled as an external module. Reviewed-by: Andrew Lunn Signed-off-by: Pali Rohár Fixes: 92ce45fb875d7 ("cpufreq: Add DVFS support for Armada 37xx") [ Viresh: Added __maybe_unused ] Signed-off-by: Viresh Kumar --- drivers/cpufreq/armada-37xx-cpufreq.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/cpufreq/armada-37xx-cpufreq.c b/drivers/cpufreq/armada-37xx-cpufreq.c index df1c941260d1..b4af4094309b 100644 --- a/drivers/cpufreq/armada-37xx-cpufreq.c +++ b/drivers/cpufreq/armada-37xx-cpufreq.c @@ -484,6 +484,12 @@ remove_opp: /* late_initcall, to guarantee the driver is loaded after A37xx clock driver */ late_initcall(armada37xx_cpufreq_driver_init); +static const struct of_device_id __maybe_unused armada37xx_cpufreq_of_match[] = { + { .compatible = "marvell,armada-3700-nb-pm" }, + { }, +}; +MODULE_DEVICE_TABLE(of, armada37xx_cpufreq_of_match); + MODULE_AUTHOR("Gregory CLEMENT "); MODULE_DESCRIPTION("Armada 37xx cpufreq driver"); MODULE_LICENSE("GPL"); From bd74e286b354134863ce633747fa4bfe541e440a Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Tue, 8 Sep 2020 13:27:12 +0530 Subject: [PATCH 123/265] cpufreq: qcom-hw: Make use of cpufreq driver_data for passing pdev Get rid of global_pdev pointer and make use of cpufreq driver_data for passing the reference of pdev. This aligns with what other cpufreq drivers are doing. Signed-off-by: Manivannan Sadhasivam Signed-off-by: Viresh Kumar --- drivers/cpufreq/qcom-cpufreq-hw.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 3fb044b907a8..ccea34f61152 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -30,7 +30,6 @@ #define REG_PERF_STATE 0x920 static unsigned long cpu_hw_rate, xo_rate; -static struct platform_device *global_pdev; static bool icc_scaling_enabled; static int qcom_cpufreq_set_bw(struct cpufreq_policy *policy, @@ -240,7 +239,8 @@ static void qcom_get_related_cpus(int index, struct cpumask *m) static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) { - struct device *dev = &global_pdev->dev; + struct platform_device *pdev = cpufreq_get_driver_data(); + struct device *dev = &pdev->dev; struct of_phandle_args args; struct device_node *cpu_np; struct device *cpu_dev; @@ -267,7 +267,7 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) index = args.args[0]; - res = platform_get_resource(global_pdev, IORESOURCE_MEM, index); + res = platform_get_resource(pdev, IORESOURCE_MEM, index); if (!res) return -ENODEV; @@ -316,11 +316,12 @@ static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) { struct device *cpu_dev = get_cpu_device(policy->cpu); void __iomem *base = policy->driver_data - REG_PERF_STATE; + struct platform_device *pdev = cpufreq_get_driver_data(); dev_pm_opp_remove_all_dynamic(cpu_dev); dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); kfree(policy->freq_table); - devm_iounmap(&global_pdev->dev, base); + devm_iounmap(&pdev->dev, base); return 0; } @@ -365,7 +366,7 @@ static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev) cpu_hw_rate = clk_get_rate(clk) / CLK_HW_DIV; clk_put(clk); - global_pdev = pdev; + cpufreq_qcom_hw_driver.driver_data = pdev; /* Check for optional interconnect paths on CPU0 */ cpu_dev = get_cpu_device(0); From 75319b4600e7b7adfec80ea0a3ccc07f22290c89 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Tue, 15 Sep 2020 12:54:19 +0530 Subject: [PATCH 124/265] dt-bindings: cpufreq: cpufreq-qcom-hw: Document Qcom EPSS compatible The hardware block which carries out CPUFreq operations on SM8250 SoC is called EPSS. Hence, document the compatible. Signed-off-by: Manivannan Sadhasivam Reviewed-by: Amit Kucheria Reviewed-by: Bjorn Andersson Acked-by: Rob Herring Signed-off-by: Viresh Kumar --- Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.txt b/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.txt index 33856947c561..9299028ee712 100644 --- a/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.txt +++ b/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.txt @@ -8,7 +8,7 @@ Properties: - compatible Usage: required Value type: - Definition: must be "qcom,cpufreq-hw". + Definition: must be "qcom,cpufreq-hw" or "qcom,cpufreq-epss". - clocks Usage: required From f17b3e44320b4079e69135c63d1e416b0703a21e Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Tue, 15 Sep 2020 12:54:21 +0530 Subject: [PATCH 125/265] cpufreq: qcom-hw: Use devm_platform_ioremap_resource() to simplify code devm_platform_ioremap_resource() is the combination of platform_get_resource() and devm_ioremap_resource(). Hence, use it to simplify the code a bit. Signed-off-by: Manivannan Sadhasivam Reviewed-by: Amit Kucheria Reviewed-by: Bjorn Andersson Signed-off-by: Viresh Kumar --- drivers/cpufreq/qcom-cpufreq-hw.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index ccea34f61152..8a303783927f 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -244,7 +244,6 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) struct of_phandle_args args; struct device_node *cpu_np; struct device *cpu_dev; - struct resource *res; void __iomem *base; int ret, index; @@ -267,13 +266,9 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) index = args.args[0]; - res = platform_get_resource(pdev, IORESOURCE_MEM, index); - if (!res) - return -ENODEV; - - base = devm_ioremap(dev, res->start, resource_size(res)); - if (!base) - return -ENOMEM; + base = devm_platform_ioremap_resource(pdev, index); + if (IS_ERR(base)) + return PTR_ERR(base); /* HW should be in enabled state to proceed */ if (!(readl_relaxed(base + REG_ENABLE) & 0x1)) { From dcd1fd724c19fe964e6415cb161ca58ef99f86ef Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Tue, 15 Sep 2020 12:54:22 +0530 Subject: [PATCH 126/265] cpufreq: qcom-hw: Use of_device_get_match_data for offsets and row size For preparing the driver to handle further SoC revisions, let's use the of_device_get_match_data() API for getting the device specific offsets and row size instead of defining them globally. Signed-off-by: Manivannan Sadhasivam Reviewed-by: Bjorn Andersson Signed-off-by: Viresh Kumar --- drivers/cpufreq/qcom-cpufreq-hw.c | 89 ++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 30 deletions(-) diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 8a303783927f..e3c46984a037 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -19,15 +19,21 @@ #define LUT_L_VAL GENMASK(7, 0) #define LUT_CORE_COUNT GENMASK(18, 16) #define LUT_VOLT GENMASK(11, 0) -#define LUT_ROW_SIZE 32 #define CLK_HW_DIV 2 #define LUT_TURBO_IND 1 -/* Register offsets */ -#define REG_ENABLE 0x0 -#define REG_FREQ_LUT 0x110 -#define REG_VOLT_LUT 0x114 -#define REG_PERF_STATE 0x920 +struct qcom_cpufreq_soc_data { + u32 reg_enable; + u32 reg_freq_lut; + u32 reg_volt_lut; + u32 reg_perf_state; + u8 lut_row_size; +}; + +struct qcom_cpufreq_data { + void __iomem *base; + const struct qcom_cpufreq_soc_data *soc_data; +}; static unsigned long cpu_hw_rate, xo_rate; static bool icc_scaling_enabled; @@ -76,10 +82,11 @@ static int qcom_cpufreq_update_opp(struct device *cpu_dev, static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy, unsigned int index) { - void __iomem *perf_state_reg = policy->driver_data; + struct qcom_cpufreq_data *data = policy->driver_data; + const struct qcom_cpufreq_soc_data *soc_data = data->soc_data; unsigned long freq = policy->freq_table[index].frequency; - writel_relaxed(index, perf_state_reg); + writel_relaxed(index, data->base + soc_data->reg_perf_state); if (icc_scaling_enabled) qcom_cpufreq_set_bw(policy, freq); @@ -91,7 +98,8 @@ static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy, static unsigned int qcom_cpufreq_hw_get(unsigned int cpu) { - void __iomem *perf_state_reg; + struct qcom_cpufreq_data *data; + const struct qcom_cpufreq_soc_data *soc_data; struct cpufreq_policy *policy; unsigned int index; @@ -99,9 +107,10 @@ static unsigned int qcom_cpufreq_hw_get(unsigned int cpu) if (!policy) return 0; - perf_state_reg = policy->driver_data; + data = policy->driver_data; + soc_data = data->soc_data; - index = readl_relaxed(perf_state_reg); + index = readl_relaxed(data->base + soc_data->reg_perf_state); index = min(index, LUT_MAX_ENTRIES - 1); return policy->freq_table[index].frequency; @@ -110,12 +119,13 @@ static unsigned int qcom_cpufreq_hw_get(unsigned int cpu) static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq) { - void __iomem *perf_state_reg = policy->driver_data; + struct qcom_cpufreq_data *data = policy->driver_data; + const struct qcom_cpufreq_soc_data *soc_data = data->soc_data; unsigned int index; unsigned long freq; index = policy->cached_resolved_idx; - writel_relaxed(index, perf_state_reg); + writel_relaxed(index, data->base + soc_data->reg_perf_state); freq = policy->freq_table[index].frequency; arch_set_freq_scale(policy->related_cpus, freq, @@ -125,8 +135,7 @@ static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy, } static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, - struct cpufreq_policy *policy, - void __iomem *base) + struct cpufreq_policy *policy) { u32 data, src, lval, i, core_count, prev_freq = 0, freq; u32 volt; @@ -134,6 +143,8 @@ static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, struct dev_pm_opp *opp; unsigned long rate; int ret; + struct qcom_cpufreq_data *drv_data = policy->driver_data; + const struct qcom_cpufreq_soc_data *soc_data = drv_data->soc_data; table = kcalloc(LUT_MAX_ENTRIES + 1, sizeof(*table), GFP_KERNEL); if (!table) @@ -160,14 +171,14 @@ static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, } for (i = 0; i < LUT_MAX_ENTRIES; i++) { - data = readl_relaxed(base + REG_FREQ_LUT + - i * LUT_ROW_SIZE); + data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut + + i * soc_data->lut_row_size); src = FIELD_GET(LUT_SRC, data); lval = FIELD_GET(LUT_L_VAL, data); core_count = FIELD_GET(LUT_CORE_COUNT, data); - data = readl_relaxed(base + REG_VOLT_LUT + - i * LUT_ROW_SIZE); + data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut + + i * soc_data->lut_row_size); volt = FIELD_GET(LUT_VOLT, data) * 1000; if (src) @@ -237,6 +248,20 @@ static void qcom_get_related_cpus(int index, struct cpumask *m) } } +static const struct qcom_cpufreq_soc_data qcom_soc_data = { + .reg_enable = 0x0, + .reg_freq_lut = 0x110, + .reg_volt_lut = 0x114, + .reg_perf_state = 0x920, + .lut_row_size = 32, +}; + +static const struct of_device_id qcom_cpufreq_hw_match[] = { + { .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data }, + {} +}; +MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match); + static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) { struct platform_device *pdev = cpufreq_get_driver_data(); @@ -245,6 +270,7 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) struct device_node *cpu_np; struct device *cpu_dev; void __iomem *base; + struct qcom_cpufreq_data *data; int ret, index; cpu_dev = get_cpu_device(policy->cpu); @@ -270,8 +296,17 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) if (IS_ERR(base)) return PTR_ERR(base); + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) { + ret = -ENOMEM; + goto error; + } + + data->soc_data = of_device_get_match_data(&pdev->dev); + data->base = base; + /* HW should be in enabled state to proceed */ - if (!(readl_relaxed(base + REG_ENABLE) & 0x1)) { + if (!(readl_relaxed(base + data->soc_data->reg_enable) & 0x1)) { dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index); ret = -ENODEV; goto error; @@ -284,9 +319,9 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) goto error; } - policy->driver_data = base + REG_PERF_STATE; + policy->driver_data = data; - ret = qcom_cpufreq_hw_read_lut(cpu_dev, policy, base); + ret = qcom_cpufreq_hw_read_lut(cpu_dev, policy); if (ret) { dev_err(dev, "Domain-%d failed to read LUT\n", index); goto error; @@ -310,13 +345,13 @@ error: static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) { struct device *cpu_dev = get_cpu_device(policy->cpu); - void __iomem *base = policy->driver_data - REG_PERF_STATE; + struct qcom_cpufreq_data *data = policy->driver_data; struct platform_device *pdev = cpufreq_get_driver_data(); dev_pm_opp_remove_all_dynamic(cpu_dev); dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); kfree(policy->freq_table); - devm_iounmap(&pdev->dev, base); + devm_iounmap(&pdev->dev, data->base); return 0; } @@ -386,12 +421,6 @@ static int qcom_cpufreq_hw_driver_remove(struct platform_device *pdev) return cpufreq_unregister_driver(&cpufreq_qcom_hw_driver); } -static const struct of_device_id qcom_cpufreq_hw_match[] = { - { .compatible = "qcom,cpufreq-hw" }, - {} -}; -MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match); - static struct platform_driver qcom_cpufreq_hw_driver = { .probe = qcom_cpufreq_hw_driver_probe, .remove = qcom_cpufreq_hw_driver_remove, From 49b59f4c358cf71cb4e413749cd855478547c2ca Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Tue, 15 Sep 2020 12:54:23 +0530 Subject: [PATCH 127/265] cpufreq: qcom-hw: Add cpufreq support for SM8250 SoC SM8250 SoC uses EPSS block for carrying out the cpufreq duties. Hence, add support for it in the driver with relevant dev data. Signed-off-by: Manivannan Sadhasivam Reviewed-by: Amit Kucheria Reviewed-by: Bjorn Andersson Signed-off-by: Viresh Kumar --- drivers/cpufreq/qcom-cpufreq-hw.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index e3c46984a037..c485be839172 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -256,8 +256,17 @@ static const struct qcom_cpufreq_soc_data qcom_soc_data = { .lut_row_size = 32, }; +static const struct qcom_cpufreq_soc_data epss_soc_data = { + .reg_enable = 0x0, + .reg_freq_lut = 0x100, + .reg_volt_lut = 0x200, + .reg_perf_state = 0x320, + .lut_row_size = 4, +}; + static const struct of_device_id qcom_cpufreq_hw_match[] = { { .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data }, + { .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data }, {} }; MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match); From bc9b9c5ab9d8d16157737db539929d57562926e9 Mon Sep 17 00:00:00 2001 From: Matthias Kaehlcke Date: Tue, 15 Sep 2020 10:10:54 -0700 Subject: [PATCH 128/265] cpufreq: qcom: Don't add frequencies without an OPP The driver currently adds all frequencies from the hardware LUT to the cpufreq table, regardless of whether the corresponding OPP exists. This prevents devices from disabling certain OPPs through the device tree and can result in CPU frequencies for which the interconnect bandwidth can't be adjusted. Only add frequencies with an OPP entry. Fixes: 55538fbc79e9 ("cpufreq: qcom: Read voltage LUT and populate OPP") Signed-off-by: Matthias Kaehlcke Signed-off-by: Viresh Kumar --- drivers/cpufreq/qcom-cpufreq-hw.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index c485be839172..17fbb8cf36a1 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -187,10 +187,15 @@ static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, freq = cpu_hw_rate / 1000; if (freq != prev_freq && core_count != LUT_TURBO_IND) { - table[i].frequency = freq; - qcom_cpufreq_update_opp(cpu_dev, freq, volt); - dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i, + if (!qcom_cpufreq_update_opp(cpu_dev, freq, volt)) { + table[i].frequency = freq; + dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i, freq, core_count); + } else { + dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", freq); + table[i].frequency = CPUFREQ_ENTRY_INVALID; + } + } else if (core_count == LUT_TURBO_IND) { table[i].frequency = CPUFREQ_ENTRY_INVALID; } @@ -207,9 +212,13 @@ static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, * as the boost frequency */ if (prev->frequency == CPUFREQ_ENTRY_INVALID) { - prev->frequency = prev_freq; - prev->flags = CPUFREQ_BOOST_FREQ; - qcom_cpufreq_update_opp(cpu_dev, prev_freq, volt); + if (!qcom_cpufreq_update_opp(cpu_dev, prev_freq, volt)) { + prev->frequency = prev_freq; + prev->flags = CPUFREQ_BOOST_FREQ; + } else { + dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", + freq); + } } break; From 2ce6324eadb014136a4baaf7a174f47d771364a0 Mon Sep 17 00:00:00 2001 From: Srinivas Pandruvada Date: Tue, 15 Sep 2020 16:18:14 -0700 Subject: [PATCH 129/265] ACPI: DPTF: Add PCH FIVR participant driver This driver adds support for Dynamic Platform and Thermal Framework (DPTF) PCH (Platform Controller Hub) FIVR (Fully Integrated Voltage Regulator) participant support. This participant is responsible for exposing platform telemetry and control for: freq_mhz_high_clock freq_mhz_low_clock These attributes are used to get and set PCH FIVR switching frequency for thermal and radio frequency interference mitigation. Refer to Documentation/ABI/testing/sysfs-platform-dptf for ABI details. ACPI methods description used in this driver: RFC0: This ACPI method to set PCH FIVR switching frequency when FIVR clock is 19.2MHz or 24MHz. The ACPI method takes a DWORD value. GFC0: This ACPI method to get PCH FIVR switching frequency when FIVR clock is 19.2MHz or 24MHz. RFC1: This ACPI method to set PCH FIVR switching frequency when FIVR clock is 38.4MHz. The ACPI method takes a DWORD value. GFC1: This ACPI method to get PCH FIVR switching frequency when FIVR clock is 38.4MHz. Signed-off-by: Srinivas Pandruvada Signed-off-by: Rafael J. Wysocki --- Documentation/ABI/testing/sysfs-platform-dptf | 16 +++ drivers/acpi/dptf/Kconfig | 14 ++ drivers/acpi/dptf/Makefile | 1 + drivers/acpi/dptf/dptf_pch_fivr.c | 126 ++++++++++++++++++ drivers/acpi/dptf/int340x_thermal.c | 1 + 5 files changed, 158 insertions(+) create mode 100644 drivers/acpi/dptf/dptf_pch_fivr.c diff --git a/Documentation/ABI/testing/sysfs-platform-dptf b/Documentation/ABI/testing/sysfs-platform-dptf index eeed81ca6949..2cbc660d163b 100644 --- a/Documentation/ABI/testing/sysfs-platform-dptf +++ b/Documentation/ABI/testing/sysfs-platform-dptf @@ -92,3 +92,19 @@ Contact: linux-acpi@vger.kernel.org Description: (RO) The battery discharge current capability obtained from battery fuel gauge in milli Amps. + +What: /sys/bus/platform/devices/INTC1045:00/pch_fivr_switch_frequency/freq_mhz_low_clock +Date: November, 2020 +KernelVersion: v5.10 +Contact: linux-acpi@vger.kernel.org +Description: + (RW) The PCH FIVR (Fully Integrated Voltage Regulator) switching frequency in MHz, + when FIVR clock is 19.2MHz or 24MHz. + +What: /sys/bus/platform/devices/INTC1045:00/pch_fivr_switch_frequency/freq_mhz_high_clock +Date: November, 2020 +KernelVersion: v5.10 +Contact: linux-acpi@vger.kernel.org +Description: + (RW) The PCH FIVR (Fully Integrated Voltage Regulator) switching frequency in MHz, + when FIVR clock is 38.4MHz. diff --git a/drivers/acpi/dptf/Kconfig b/drivers/acpi/dptf/Kconfig index 90a2fd979282..51f06f36cafa 100644 --- a/drivers/acpi/dptf/Kconfig +++ b/drivers/acpi/dptf/Kconfig @@ -14,3 +14,17 @@ config DPTF_POWER To compile this driver as a module, choose M here: the module will be called dptf_power. + +config DPTF_PCH_FIVR + tristate "DPTF PCH FIVR Participant" + depends on X86 + help + This driver adds support for Dynamic Platform and Thermal Framework + (DPTF) PCH FIVR Participant device support. This driver allows to + switch PCH FIVR (Fully Integrated Voltage Regulator) frequency. + This participant is responsible for exposing: + freq_mhz_low_clock + freq_mhz_high_clock + + To compile this driver as a module, choose M here: + the module will be called dptf_pch_fivr. diff --git a/drivers/acpi/dptf/Makefile b/drivers/acpi/dptf/Makefile index 1a9b0a2b25bf..297340682f66 100644 --- a/drivers/acpi/dptf/Makefile +++ b/drivers/acpi/dptf/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_ACPI) += int340x_thermal.o obj-$(CONFIG_DPTF_POWER) += dptf_power.o +obj-$(CONFIG_DPTF_PCH_FIVR) += dptf_pch_fivr.o diff --git a/drivers/acpi/dptf/dptf_pch_fivr.c b/drivers/acpi/dptf/dptf_pch_fivr.c new file mode 100644 index 000000000000..4ab288827747 --- /dev/null +++ b/drivers/acpi/dptf/dptf_pch_fivr.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * dptf_pch_fivr: DPTF PCH FIVR Participant driver + * Copyright (c) 2020, Intel Corporation. + */ + +#include +#include +#include +#include + +/* + * Presentation of attributes which are defined for INT1045 + * They are: + * freq_mhz_low_clock : Set PCH FIVR switching freq for + * FIVR clock 19.2MHz and 24MHz + * freq_mhz_high_clock : Set PCH FIVR switching freq for + * FIVR clock 38.4MHz + */ +#define PCH_FIVR_SHOW(name, method) \ +static ssize_t name##_show(struct device *dev,\ + struct device_attribute *attr,\ + char *buf)\ +{\ + struct acpi_device *acpi_dev = dev_get_drvdata(dev);\ + unsigned long long val;\ + acpi_status status;\ +\ + status = acpi_evaluate_integer(acpi_dev->handle, #method,\ + NULL, &val);\ + if (ACPI_SUCCESS(status))\ + return sprintf(buf, "%d\n", (int)val);\ + else\ + return -EINVAL;\ +} + +#define PCH_FIVR_STORE(name, method) \ +static ssize_t name##_store(struct device *dev,\ + struct device_attribute *attr,\ + const char *buf, size_t count)\ +{\ + struct acpi_device *acpi_dev = dev_get_drvdata(dev);\ + acpi_status status;\ + u32 val;\ +\ + if (kstrtouint(buf, 0, &val) < 0)\ + return -EINVAL;\ +\ + status = acpi_execute_simple_method(acpi_dev->handle, #method, val);\ + if (ACPI_SUCCESS(status))\ + return count;\ +\ + return -EINVAL;\ +} + +PCH_FIVR_SHOW(freq_mhz_low_clock, GFC0) +PCH_FIVR_SHOW(freq_mhz_high_clock, GFC1) +PCH_FIVR_STORE(freq_mhz_low_clock, RFC0) +PCH_FIVR_STORE(freq_mhz_high_clock, RFC1) + +static DEVICE_ATTR_RW(freq_mhz_low_clock); +static DEVICE_ATTR_RW(freq_mhz_high_clock); + +static struct attribute *fivr_attrs[] = { + &dev_attr_freq_mhz_low_clock.attr, + &dev_attr_freq_mhz_high_clock.attr, + NULL +}; + +static const struct attribute_group pch_fivr_attribute_group = { + .attrs = fivr_attrs, + .name = "pch_fivr_switch_frequency" +}; + +static int pch_fivr_add(struct platform_device *pdev) +{ + struct acpi_device *acpi_dev; + unsigned long long ptype; + acpi_status status; + int result; + + acpi_dev = ACPI_COMPANION(&(pdev->dev)); + if (!acpi_dev) + return -ENODEV; + + status = acpi_evaluate_integer(acpi_dev->handle, "PTYP", NULL, &ptype); + if (ACPI_FAILURE(status) || ptype != 0x05) + return -ENODEV; + + result = sysfs_create_group(&pdev->dev.kobj, + &pch_fivr_attribute_group); + if (result) + return result; + + platform_set_drvdata(pdev, acpi_dev); + + return 0; +} + +static int pch_fivr_remove(struct platform_device *pdev) +{ + sysfs_remove_group(&pdev->dev.kobj, &pch_fivr_attribute_group); + + return 0; +} + +static const struct acpi_device_id pch_fivr_device_ids[] = { + {"INTC1045", 0}, + {"", 0}, +}; +MODULE_DEVICE_TABLE(acpi, pch_fivr_device_ids); + +static struct platform_driver pch_fivr_driver = { + .probe = pch_fivr_add, + .remove = pch_fivr_remove, + .driver = { + .name = "DPTF PCH FIVR", + .acpi_match_table = pch_fivr_device_ids, + }, +}; + +module_platform_driver(pch_fivr_driver); + +MODULE_AUTHOR("Srinivas Pandruvada "); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("ACPI DPTF PCH FIVR driver"); diff --git a/drivers/acpi/dptf/int340x_thermal.c b/drivers/acpi/dptf/int340x_thermal.c index bc71a6a60334..8d420c7e7178 100644 --- a/drivers/acpi/dptf/int340x_thermal.c +++ b/drivers/acpi/dptf/int340x_thermal.c @@ -27,6 +27,7 @@ static const struct acpi_device_id int340x_thermal_device_ids[] = { {"INTC1040"}, {"INTC1043"}, {"INTC1044"}, + {"INTC1045"}, {"INTC1047"}, {""}, }; From f4f803984c3685f416a74e9e2fa7d39bdafbe02b Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Tue, 15 Sep 2020 08:53:16 +0100 Subject: [PATCH 130/265] objtool: Remove useless tests before save_reg() save_reg already checks that the register being saved does not already have a saved state. Remove redundant checks before processing a register storing operation. Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- tools/objtool/check.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 4e2f703b6a25..fd2edab8e672 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2030,7 +2030,7 @@ static int update_cfi_state(struct instruction *insn, struct cfi_state *cfi, /* drap: push %rbp */ cfi->stack_size = 0; - } else if (regs[op->src.reg].base == CFI_UNDEFINED) { + } else { /* drap: push %reg */ save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size); @@ -2059,9 +2059,7 @@ static int update_cfi_state(struct instruction *insn, struct cfi_state *cfi, /* save drap offset so we know when to restore it */ cfi->drap_offset = op->dest.offset; - } - - else if (regs[op->src.reg].base == CFI_UNDEFINED) { + } else { /* drap: mov reg, disp(%rbp) */ save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset); From fb136219f0e2b417d84e67b2a3adc1f933997d04 Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Tue, 15 Sep 2020 08:53:17 +0100 Subject: [PATCH 131/265] objtool: Ignore unreachable fake jumps It is possible for alternative code to unconditionally jump out of the alternative region. In such a case, if a fake jump is added at the end of the alternative instructions, the fake jump will never be reached. Since the fake jump is just a mean to make sure code validation does not go beyond the set of alternatives, reaching it is not a requirement. Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- tools/objtool/check.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index fd2edab8e672..cd7c6698d316 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2648,6 +2648,9 @@ static bool ignore_unreachable_insn(struct instruction *insn) !strcmp(insn->sec->name, ".altinstr_aux")) return true; + if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->offset == FAKE_JUMP_OFFSET) + return true; + if (!insn->func) return false; From 0a10d3fe3e5c601031676e81b41fb2977650b4d4 Mon Sep 17 00:00:00 2001 From: Ionela Voinescu Date: Tue, 1 Sep 2020 21:55:45 +0100 Subject: [PATCH 132/265] arch_topology: validate input frequencies to arch_set_freq_scale() The current frequency passed to arch_set_freq_scale() could end up being 0, signaling an error in setting a new frequency. Also, if the maximum frequency in 0, this will result in a division by 0 error. Therefore, validate these input values before using them for the setting of the frequency scale factor. Signed-off-by: Ionela Voinescu Acked-by: Viresh Kumar Reviewed-by: Sudeep Holla Signed-off-by: Rafael J. Wysocki --- drivers/base/arch_topology.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 75f72d684294..42a08ef693ae 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -33,6 +33,9 @@ void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, unsigned long scale; int i; + if (WARN_ON_ONCE(!cur_freq || !max_freq)) + return; + /* * If the use of counters for FIE is enabled, just return as we don't * want to update the scale factor with information from CPUFREQ. From 1a0419b0db4647107a9cf4a28b331c343c0378a0 Mon Sep 17 00:00:00 2001 From: Ionela Voinescu Date: Tue, 1 Sep 2020 21:55:46 +0100 Subject: [PATCH 133/265] cpufreq: move invariance setter calls in cpufreq core To properly scale its per-entity load-tracking signals, the task scheduler needs to be given a frequency scale factor, i.e. some image of the current frequency the CPU is running at. Currently, this scale can be computed either by using counters (APERF/MPERF on x86, AMU on arm64), or by piggy-backing on the frequency selection done by cpufreq. For the latter, drivers have to explicitly set the scale factor themselves, despite it being purely boiler-plate code: the required information depends entirely on the kind of frequency switch callback implemented by the driver, i.e. either of: target_index(), target(), fast_switch() and setpolicy(). The fitness of those callbacks with regard to driving the Frequency Invariance Engine (FIE) is studied below: target_index() ============== Documentation states that the chosen frequency "must be determined by freq_table[index].frequency". It isn't clear if it *has* to be that frequency, or if it can use that frequency value to do some computation that ultimately leads to a different frequency selection. All drivers go for the former, while the vexpress-spc-cpufreq has an atypical implementation which is handled separately. Therefore, the hook works on the assumption the core can use freq_table[index].frequency. target() ======= This has been flagged as deprecated since: commit 9c0ebcf78fde ("cpufreq: Implement light weight ->target_index() routine") It also doesn't have that many users: gx-suspmod.c:439: .target = cpufreq_gx_target, s3c24xx-cpufreq.c:428: .target = s3c_cpufreq_target, intel_pstate.c:2528: .target = intel_cpufreq_target, cppc_cpufreq.c:401: .target = cppc_cpufreq_set_target, cpufreq-nforce2.c:371: .target = nforce2_target, sh-cpufreq.c:163: .target = sh_cpufreq_target, pcc-cpufreq.c:573: .target = pcc_cpufreq_target, Similarly to the path taken for target_index() calls in the cpufreq core during a frequency change, all of the drivers above will mark the end of a frequency change by a call to cpufreq_freq_transition_end(). Therefore, cpufreq_freq_transition_end() can be used as the location for the arch_set_freq_scale() call to potentially inform the scheduler of the frequency change. This change maintains the previous functionality for the drivers that implement the target_index() callback, while also adding support for the few drivers that implement the deprecated target() callback. fast_switch() ============= This callback *has* to return the frequency that was selected. setpolicy() =========== This callback does not have any designated way of informing what was the end choice. But there are only two drivers using setpolicy(), and none of them have current FIE support: drivers/cpufreq/longrun.c:281: .setpolicy = longrun_set_policy, drivers/cpufreq/intel_pstate.c:2215: .setpolicy = intel_pstate_set_policy, The intel_pstate is known to use counter-driven frequency invariance. Conclusion ========== Given that the significant majority of current FIE enabled drivers use callbacks that lend themselves to triggering the setting of the FIE scale factor in a generic way, move the invariance setter calls to cpufreq core. As a result of setting the frequency scale factor in cpufreq core, after callbacks that lend themselves to trigger it, remove this functionality from the driver side. To be noted that despite marking a successful frequency change, many cpufreq drivers will consider the new frequency as the requested frequency, although this is might not be the one granted by the hardware. Therefore, the call to arch_set_freq_scale() is a "best effort" one, and it is up to the architecture if the new frequency is used in the new frequency scale factor setting (determined by the implementation of arch_set_freq_scale()) or eventually used by the scheduler (determined by the implementation of arch_scale_freq_capacity()). The architecture is in a better position to decide if it has better methods to obtain more accurate information regarding the current frequency and use that information instead (for example, the use of counters). Also, the implementation to arch_set_freq_scale() will now have to handle error conditions (current frequency == 0) in order to prevent the overhead in cpufreq core when the default arch_set_freq_scale() implementation is used. Signed-off-by: Ionela Voinescu Suggested-by: Valentin Schneider Acked-by: Viresh Kumar Acked-by: Sudeep Holla Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq-dt.c | 10 +--------- drivers/cpufreq/cpufreq.c | 14 ++++++++++++-- drivers/cpufreq/qcom-cpufreq-hw.c | 9 +-------- drivers/cpufreq/scmi-cpufreq.c | 12 ++---------- drivers/cpufreq/scpi-cpufreq.c | 6 +----- drivers/cpufreq/vexpress-spc-cpufreq.c | 12 ++---------- 6 files changed, 19 insertions(+), 44 deletions(-) diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index 944d7b45afe9..9fd4ce774f12 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -40,16 +40,8 @@ static int set_target(struct cpufreq_policy *policy, unsigned int index) { struct private_data *priv = policy->driver_data; unsigned long freq = policy->freq_table[index].frequency; - int ret; - ret = dev_pm_opp_set_rate(priv->cpu_dev, freq * 1000); - - if (!ret) { - arch_set_freq_scale(policy->related_cpus, freq, - policy->cpuinfo.max_freq); - } - - return ret; + return dev_pm_opp_set_rate(priv->cpu_dev, freq * 1000); } /* diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 47aa90f9a7c2..4d5fe777184a 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -446,6 +446,10 @@ void cpufreq_freq_transition_end(struct cpufreq_policy *policy, cpufreq_notify_post_transition(policy, freqs, transition_failed); + arch_set_freq_scale(policy->related_cpus, + policy->cur, + policy->cpuinfo.max_freq); + policy->transition_ongoing = false; policy->transition_task = NULL; @@ -2056,9 +2060,15 @@ EXPORT_SYMBOL(cpufreq_unregister_notifier); unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq) { - target_freq = clamp_val(target_freq, policy->min, policy->max); + unsigned int freq; - return cpufreq_driver->fast_switch(policy, target_freq); + target_freq = clamp_val(target_freq, policy->min, policy->max); + freq = cpufreq_driver->fast_switch(policy, target_freq); + + arch_set_freq_scale(policy->related_cpus, freq, + policy->cpuinfo.max_freq); + + return freq; } EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 3fb044b907a8..aabe4306d92f 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -85,8 +85,6 @@ static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy, if (icc_scaling_enabled) qcom_cpufreq_set_bw(policy, freq); - arch_set_freq_scale(policy->related_cpus, freq, - policy->cpuinfo.max_freq); return 0; } @@ -113,16 +111,11 @@ static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy, { void __iomem *perf_state_reg = policy->driver_data; unsigned int index; - unsigned long freq; index = policy->cached_resolved_idx; writel_relaxed(index, perf_state_reg); - freq = policy->freq_table[index].frequency; - arch_set_freq_scale(policy->related_cpus, freq, - policy->cpuinfo.max_freq); - - return freq; + return policy->freq_table[index].frequency; } static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index fb42e3390377..6dd1311660b5 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -48,16 +48,11 @@ static unsigned int scmi_cpufreq_get_rate(unsigned int cpu) static int scmi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index) { - int ret; struct scmi_data *priv = policy->driver_data; struct scmi_perf_ops *perf_ops = handle->perf_ops; u64 freq = policy->freq_table[index].frequency; - ret = perf_ops->freq_set(handle, priv->domain_id, freq * 1000, false); - if (!ret) - arch_set_freq_scale(policy->related_cpus, freq, - policy->cpuinfo.max_freq); - return ret; + return perf_ops->freq_set(handle, priv->domain_id, freq * 1000, false); } static unsigned int scmi_cpufreq_fast_switch(struct cpufreq_policy *policy, @@ -67,11 +62,8 @@ static unsigned int scmi_cpufreq_fast_switch(struct cpufreq_policy *policy, struct scmi_perf_ops *perf_ops = handle->perf_ops; if (!perf_ops->freq_set(handle, priv->domain_id, - target_freq * 1000, true)) { - arch_set_freq_scale(policy->related_cpus, target_freq, - policy->cpuinfo.max_freq); + target_freq * 1000, true)) return target_freq; - } return 0; } diff --git a/drivers/cpufreq/scpi-cpufreq.c b/drivers/cpufreq/scpi-cpufreq.c index b0f5388b8854..43db05b949d9 100644 --- a/drivers/cpufreq/scpi-cpufreq.c +++ b/drivers/cpufreq/scpi-cpufreq.c @@ -47,9 +47,8 @@ static unsigned int scpi_cpufreq_get_rate(unsigned int cpu) static int scpi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index) { - unsigned long freq = policy->freq_table[index].frequency; + u64 rate = policy->freq_table[index].frequency * 1000; struct scpi_data *priv = policy->driver_data; - u64 rate = freq * 1000; int ret; ret = clk_set_rate(priv->clk, rate); @@ -60,9 +59,6 @@ scpi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index) if (clk_get_rate(priv->clk) != rate) return -EIO; - arch_set_freq_scale(policy->related_cpus, freq, - policy->cpuinfo.max_freq); - return 0; } diff --git a/drivers/cpufreq/vexpress-spc-cpufreq.c b/drivers/cpufreq/vexpress-spc-cpufreq.c index 4e8b1dee7c9a..e89b905754d2 100644 --- a/drivers/cpufreq/vexpress-spc-cpufreq.c +++ b/drivers/cpufreq/vexpress-spc-cpufreq.c @@ -182,7 +182,6 @@ static int ve_spc_cpufreq_set_target(struct cpufreq_policy *policy, { u32 cpu = policy->cpu, cur_cluster, new_cluster, actual_cluster; unsigned int freqs_new; - int ret; cur_cluster = cpu_to_cluster(cpu); new_cluster = actual_cluster = per_cpu(physical_cluster, cpu); @@ -197,15 +196,8 @@ static int ve_spc_cpufreq_set_target(struct cpufreq_policy *policy, new_cluster = A15_CLUSTER; } - ret = ve_spc_cpufreq_set_rate(cpu, actual_cluster, new_cluster, - freqs_new); - - if (!ret) { - arch_set_freq_scale(policy->related_cpus, freqs_new, - policy->cpuinfo.max_freq); - } - - return ret; + return ve_spc_cpufreq_set_rate(cpu, actual_cluster, new_cluster, + freqs_new); } static inline u32 get_table_count(struct cpufreq_frequency_table *table) From 874f635310648a5adcedbd7e02ea0555cfa1da56 Mon Sep 17 00:00:00 2001 From: Ionela Voinescu Date: Tue, 1 Sep 2020 21:55:47 +0100 Subject: [PATCH 134/265] cpufreq: report whether cpufreq supports Frequency Invariance (FI) Now that the update of the FI scale factor is done in cpufreq core for selected functions - target(), target_index() and fast_switch(), we can provide feedback to the task scheduler and architecture code on whether cpufreq supports FI. For this purpose provide an external function to expose whether the cpufreq drivers support FI, by using a static key. The logic behind the enablement of cpufreq-based invariance is as follows: - cpufreq-based invariance is disabled by default - cpufreq-based invariance is enabled if any of the callbacks above is implemented while the unsupported setpolicy() is not The cpufreq_supports_freq_invariance() function only returns whether cpufreq is instrumented with the arch_set_freq_scale() calls that result in support for frequency invariance. Due to the lack of knowledge on whether the implementation of arch_set_freq_scale() actually results in the setting of a scale factor based on cpufreq information, it is up to the architecture code to ensure the setting and provision of the scale factor to the scheduler. Signed-off-by: Ionela Voinescu Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq.c | 16 ++++++++++++++++ include/linux/cpufreq.h | 5 +++++ 2 files changed, 21 insertions(+) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 4d5fe777184a..570bf2ebe9d4 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -61,6 +61,12 @@ static struct cpufreq_driver *cpufreq_driver; static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data); static DEFINE_RWLOCK(cpufreq_driver_lock); +static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance); +bool cpufreq_supports_freq_invariance(void) +{ + return static_branch_likely(&cpufreq_freq_invariance); +} + /* Flag to suspend/resume CPUFreq governors */ static bool cpufreq_suspended; @@ -2720,6 +2726,15 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) cpufreq_driver = driver_data; write_unlock_irqrestore(&cpufreq_driver_lock, flags); + /* + * Mark support for the scheduler's frequency invariance engine for + * drivers that implement target(), target_index() or fast_switch(). + */ + if (!cpufreq_driver->setpolicy) { + static_branch_enable_cpuslocked(&cpufreq_freq_invariance); + pr_debug("supports frequency invariance"); + } + if (driver_data->setpolicy) driver_data->flags |= CPUFREQ_CONST_LOOPS; @@ -2789,6 +2804,7 @@ int cpufreq_unregister_driver(struct cpufreq_driver *driver) cpus_read_lock(); subsys_interface_unregister(&cpufreq_interface); remove_boost_sysfs_file(); + static_branch_disable_cpuslocked(&cpufreq_freq_invariance); cpuhp_remove_state_nocalls_cpuslocked(hp_online); write_lock_irqsave(&cpufreq_driver_lock, flags); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index a911e5d06845..e54767e2a68a 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -217,6 +217,7 @@ void refresh_frequency_limits(struct cpufreq_policy *policy); void cpufreq_update_policy(unsigned int cpu); void cpufreq_update_limits(unsigned int cpu); bool have_governor_per_policy(void); +bool cpufreq_supports_freq_invariance(void); struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy); void cpufreq_enable_fast_switch(struct cpufreq_policy *policy); void cpufreq_disable_fast_switch(struct cpufreq_policy *policy); @@ -237,6 +238,10 @@ static inline unsigned int cpufreq_get_hw_max_freq(unsigned int cpu) { return 0; } +static inline bool cpufreq_supports_freq_invariance(void) +{ + return false; +} static inline void disable_cpufreq(void) { } #endif From ecddc3a0d5d752071c627aa1a1d4d7b529ddae67 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Tue, 1 Sep 2020 21:55:48 +0100 Subject: [PATCH 135/265] arch_topology, cpufreq: constify arch_* cpumasks The passed cpumask arguments to arch_set_freq_scale() and arch_freq_counters_available() are only iterated over, so reflect this in the prototype. This also allows to pass system cpumasks like cpu_online_mask without getting a warning. Signed-off-by: Valentin Schneider Signed-off-by: Ionela Voinescu Acked-by: Catalin Marinas Acked-by: Viresh Kumar Reviewed-by: Sudeep Holla Signed-off-by: Rafael J. Wysocki --- arch/arm64/kernel/topology.c | 2 +- drivers/base/arch_topology.c | 4 ++-- drivers/cpufreq/cpufreq.c | 5 +++-- include/linux/arch_topology.h | 2 +- include/linux/cpufreq.h | 3 ++- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 0801a0f3c156..9a9f2b8dedf5 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -253,7 +253,7 @@ free_valid_mask: } late_initcall_sync(init_amu_fie); -bool arch_freq_counters_available(struct cpumask *cpus) +bool arch_freq_counters_available(const struct cpumask *cpus) { return amu_freq_invariant() && cpumask_subset(cpus, amu_fie_cpus); diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 42a08ef693ae..91de5331ac8a 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -21,13 +21,13 @@ #include #include -__weak bool arch_freq_counters_available(struct cpumask *cpus) +__weak bool arch_freq_counters_available(const struct cpumask *cpus) { return false; } DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE; -void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, +void arch_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq, unsigned long max_freq) { unsigned long scale; diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 570bf2ebe9d4..2ea245a6c0c0 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -160,8 +160,9 @@ u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) } EXPORT_SYMBOL_GPL(get_cpu_idle_time); -__weak void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, - unsigned long max_freq) +__weak void arch_set_freq_scale(const struct cpumask *cpus, + unsigned long cur_freq, + unsigned long max_freq) { } EXPORT_SYMBOL_GPL(arch_set_freq_scale); diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index 69b1dabe39dc..810c83336257 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -30,7 +30,7 @@ static inline unsigned long topology_get_freq_scale(int cpu) return per_cpu(freq_scale, cpu); } -bool arch_freq_counters_available(struct cpumask *cpus); +bool arch_freq_counters_available(const struct cpumask *cpus); DECLARE_PER_CPU(unsigned long, thermal_pressure); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index e54767e2a68a..9f779fbdbe7b 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -1011,7 +1011,8 @@ static inline void sched_cpufreq_governor_change(struct cpufreq_policy *policy, extern void arch_freq_prepare_all(void); extern unsigned int arch_freq_get_on_cpu(int cpu); -extern void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, +extern void arch_set_freq_scale(const struct cpumask *cpus, + unsigned long cur_freq, unsigned long max_freq); /* the following are really really optional */ From 15e5d5b45b2b7072214af519357a1c0af078c50b Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Tue, 1 Sep 2020 21:55:49 +0100 Subject: [PATCH 136/265] arch_topology, arm, arm64: define arch_scale_freq_invariant() arch_scale_freq_invariant() is used by schedutil to determine whether the scheduler's load-tracking signals are frequency invariant. Its definition is overridable, though by default it is hardcoded to 'true' if arch_scale_freq_capacity() is defined ('false' otherwise). This behaviour is not overridden on arm, arm64 and other users of the generic arch topology driver, which is somewhat precarious: arch_scale_freq_capacity() will always be defined, yet not all cpufreq drivers are guaranteed to drive the frequency invariance scale factor setting. In other words, the load-tracking signals may very well *not* be frequency invariant. Now that cpufreq can be queried on whether the current driver is driving the Frequency Invariance (FI) scale setting, the current situation can be improved. This combines the query of whether cpufreq supports the setting of the frequency scale factor, with whether all online CPUs are counter-based FI enabled. While cpufreq FI enablement applies at system level, for all CPUs, counter-based FI support could also be used for only a subset of CPUs to set the invariance scale factor. Therefore, if cpufreq-based FI support is present, we consider the system to be invariant. If missing, we require all online CPUs to be counter-based FI enabled in order for the full system to be considered invariant. If the system ends up not being invariant, a new condition is needed in the counter initialization code that disables all scale factor setting based on counters. Precedence of counters over cpufreq use is not important here. The invariant status is only given to the system if all CPUs have at least one method of setting the frequency scale factor. Signed-off-by: Valentin Schneider Signed-off-by: Ionela Voinescu Acked-by: Catalin Marinas Acked-by: Viresh Kumar Reviewed-by: Sudeep Holla Signed-off-by: Rafael J. Wysocki --- arch/arm/include/asm/topology.h | 1 + arch/arm64/include/asm/topology.h | 1 + arch/arm64/kernel/topology.c | 7 +++++++ drivers/base/arch_topology.c | 6 ++++++ include/linux/arch_topology.h | 2 ++ 5 files changed, 17 insertions(+) diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index e0593cf095d0..9219e67befbe 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -9,6 +9,7 @@ /* Replace task scheduler's default frequency-invariant accounting */ #define arch_scale_freq_capacity topology_get_freq_scale +#define arch_scale_freq_invariant topology_scale_freq_invariant /* Replace task scheduler's default cpu-invariant accounting */ #define arch_scale_cpu_capacity topology_get_cpu_scale diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index e042f6527981..7cb519473fbd 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -27,6 +27,7 @@ void topology_scale_freq_tick(void); /* Replace task scheduler's default frequency-invariant accounting */ #define arch_scale_freq_capacity topology_get_freq_scale +#define arch_scale_freq_invariant topology_scale_freq_invariant /* Replace task scheduler's default cpu-invariant accounting */ #define arch_scale_cpu_capacity topology_get_cpu_scale diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 9a9f2b8dedf5..4064d39bb66d 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -246,6 +246,13 @@ static int __init init_amu_fie(void) static_branch_enable(&amu_fie_key); } + /* + * If the system is not fully invariant after AMU init, disable + * partial use of counters for frequency invariance. + */ + if (!topology_scale_freq_invariant()) + static_branch_disable(&amu_fie_key); + free_valid_mask: free_cpumask_var(valid_cpus); diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 91de5331ac8a..89cae168b076 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -21,6 +21,12 @@ #include #include +bool topology_scale_freq_invariant(void) +{ + return cpufreq_supports_freq_invariance() || + arch_freq_counters_available(cpu_online_mask); +} + __weak bool arch_freq_counters_available(const struct cpumask *cpus) { return false; diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index 810c83336257..083df331a3c9 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -30,6 +30,8 @@ static inline unsigned long topology_get_freq_scale(int cpu) return per_cpu(freq_scale, cpu); } +bool topology_scale_freq_invariant(void); + bool arch_freq_counters_available(const struct cpumask *cpus); DECLARE_PER_CPU(unsigned long, thermal_pressure); From 2b232a22d8225df419a92ca69ddeeb4e5fe902f7 Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Tue, 15 Sep 2020 08:53:18 +0100 Subject: [PATCH 137/265] objtool: Handle calling non-function symbols in other sections Relocation for a call destination could point to a symbol that has type STT_NOTYPE. Lookup such a symbol when no function is available. Signed-off-by: Julien Thierry Signed-off-by: Josh Poimboeuf --- tools/objtool/check.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index cd7c6698d316..a4796e32bbd1 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -815,6 +815,17 @@ static void remove_insn_ops(struct instruction *insn) } } +static struct symbol *find_call_destination(struct section *sec, unsigned long offset) +{ + struct symbol *call_dest; + + call_dest = find_func_by_offset(sec, offset); + if (!call_dest) + call_dest = find_symbol_by_offset(sec, offset); + + return call_dest; +} + /* * Find the destination instructions for all calls. */ @@ -832,9 +843,7 @@ static int add_call_destinations(struct objtool_file *file) insn->offset, insn->len); if (!reloc) { dest_off = arch_jump_destination(insn); - insn->call_dest = find_func_by_offset(insn->sec, dest_off); - if (!insn->call_dest) - insn->call_dest = find_symbol_by_offset(insn->sec, dest_off); + insn->call_dest = find_call_destination(insn->sec, dest_off); if (insn->ignore) continue; @@ -852,8 +861,8 @@ static int add_call_destinations(struct objtool_file *file) } else if (reloc->sym->type == STT_SECTION) { dest_off = arch_dest_reloc_offset(reloc->addend); - insn->call_dest = find_func_by_offset(reloc->sym->sec, - dest_off); + insn->call_dest = find_call_destination(reloc->sym->sec, + dest_off); if (!insn->call_dest) { WARN_FUNC("can't find call dest symbol at %s+0x%lx", insn->sec, insn->offset, From 14db1f0a93331d0958e90da522c429ff0890d2d6 Mon Sep 17 00:00:00 2001 From: Ilie Halip Date: Sat, 19 Sep 2020 09:41:18 +0300 Subject: [PATCH 138/265] objtool: Ignore unreachable trap after call to noreturn functions With CONFIG_UBSAN_TRAP enabled, the compiler may insert a trap instruction after a call to a noreturn function. In this case, objtool warns that the UD2 instruction is unreachable. This is a behavior seen with Clang, from the oldest version capable of building the mainline x64_64 kernel (9.0), to the latest experimental version (12.0). Objtool silences similar warnings (trap after dead end instructions), so so expand that check to include dead end functions. Cc: Nick Desaulniers Cc: Rong Chen Cc: Marco Elver Cc: Philip Li Cc: Borislav Petkov Cc: kasan-dev@googlegroups.com Cc: x86@kernel.org Cc: clang-built-linux@googlegroups.com BugLink: https://github.com/ClangBuiltLinux/linux/issues/1148 Link: https://lore.kernel.org/lkml/CAKwvOdmptEpi8fiOyWUo=AiZJiX+Z+VHJOM2buLPrWsMTwLnyw@mail.gmail.com Suggested-by: Nick Desaulniers Reviewed-by: Nick Desaulniers Tested-by: Nick Desaulniers Reported-by: kbuild test robot Signed-off-by: Ilie Halip Tested-by: Sedat Dilek Signed-off-by: Josh Poimboeuf --- tools/objtool/check.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index a4796e32bbd1..2df9f769412e 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2638,9 +2638,10 @@ static bool is_ubsan_insn(struct instruction *insn) "__ubsan_handle_builtin_unreachable")); } -static bool ignore_unreachable_insn(struct instruction *insn) +static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn) { int i; + struct instruction *prev_insn; if (insn->ignore || insn->type == INSN_NOP) return true; @@ -2668,8 +2669,11 @@ static bool ignore_unreachable_insn(struct instruction *insn) * __builtin_unreachable(). The BUG() macro has an unreachable() after * the UD2, which causes GCC's undefined trap logic to emit another UD2 * (or occasionally a JMP to UD2). + * + * It may also insert a UD2 after calling a __noreturn function. */ - if (list_prev_entry(insn, list)->dead_end && + prev_insn = list_prev_entry(insn, list); + if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) && (insn->type == INSN_BUG || (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && insn->jump_dest->type == INSN_BUG))) @@ -2796,7 +2800,7 @@ static int validate_reachable_instructions(struct objtool_file *file) return 0; for_each_insn(file, insn) { - if (insn->visited || ignore_unreachable_insn(insn)) + if (insn->visited || ignore_unreachable_insn(file, insn)) continue; WARN_FUNC("unreachable instruction", insn->sec, insn->offset); From 1170433e6611402b869c583fa1fbfd85106ff066 Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Thu, 9 Jul 2020 20:35:32 +0300 Subject: [PATCH 139/265] cpuidle: tegra: Correctly handle result of arm_cpuidle_simple_enter() The enter() callback of CPUIDLE drivers returns index of the entered idle state on success or a negative value on failure. The negative value could any negative value, i.e. it doesn't necessarily needs to be a error code. That's because CPUIDLE core only cares about the fact of failure and not about the reason of the enter() failure. Like every other enter() callback, the arm_cpuidle_simple_enter() returns the entered idle-index on success. Unlike some of other drivers, it never fails. It happened that TEGRA_C1=index=err=0 in the code of cpuidle-tegra driver, and thus, there is no problem for the cpuidle-tegra driver created by the typo in the code which assumes that the arm_cpuidle_simple_enter() returns a error code. The arm_cpuidle_simple_enter() also may return a -ENODEV error if CPU_IDLE is disabled in a kernel's config, but all CPUIDLE drivers are disabled if CPU_IDLE is disabled, including the cpuidle-tegra driver. So we can't ever see the error code from arm_cpuidle_simple_enter() today. Of course the code may get some changes in the future and then the typo may transform into a real bug, so let's correct the typo! The tegra_cpuidle_state_enter() is now changed to make it return the entered idle-index on success and negative error code on fail, which puts it on par with the arm_cpuidle_simple_enter(), making code consistent in regards to the error handling. This patch fixes a minor typo in the code, it doesn't fix any bugs. Signed-off-by: Dmitry Osipenko Reviewed-by: Jon Hunter Signed-off-by: Rafael J. Wysocki --- drivers/cpuidle/cpuidle-tegra.c | 34 +++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/drivers/cpuidle/cpuidle-tegra.c b/drivers/cpuidle/cpuidle-tegra.c index a12fb141875a..e8956706a291 100644 --- a/drivers/cpuidle/cpuidle-tegra.c +++ b/drivers/cpuidle/cpuidle-tegra.c @@ -172,7 +172,7 @@ static int tegra_cpuidle_coupled_barrier(struct cpuidle_device *dev) static int tegra_cpuidle_state_enter(struct cpuidle_device *dev, int index, unsigned int cpu) { - int ret; + int err; /* * CC6 state is the "CPU cluster power-off" state. In order to @@ -183,9 +183,9 @@ static int tegra_cpuidle_state_enter(struct cpuidle_device *dev, * CPU cores, GIC and L2 cache). */ if (index == TEGRA_CC6) { - ret = tegra_cpuidle_coupled_barrier(dev); - if (ret) - return ret; + err = tegra_cpuidle_coupled_barrier(dev); + if (err) + return err; } local_fiq_disable(); @@ -194,15 +194,15 @@ static int tegra_cpuidle_state_enter(struct cpuidle_device *dev, switch (index) { case TEGRA_C7: - ret = tegra_cpuidle_c7_enter(); + err = tegra_cpuidle_c7_enter(); break; case TEGRA_CC6: - ret = tegra_cpuidle_cc6_enter(cpu); + err = tegra_cpuidle_cc6_enter(cpu); break; default: - ret = -EINVAL; + err = -EINVAL; break; } @@ -210,7 +210,7 @@ static int tegra_cpuidle_state_enter(struct cpuidle_device *dev, tegra_pm_clear_cpu_in_lp2(); local_fiq_enable(); - return ret; + return err ?: index; } static int tegra_cpuidle_adjust_state_index(int index, unsigned int cpu) @@ -236,21 +236,27 @@ static int tegra_cpuidle_enter(struct cpuidle_device *dev, int index) { unsigned int cpu = cpu_logical_map(dev->cpu); - int err; + int ret; index = tegra_cpuidle_adjust_state_index(index, cpu); if (dev->states_usage[index].disable) return -1; if (index == TEGRA_C1) - err = arm_cpuidle_simple_enter(dev, drv, index); + ret = arm_cpuidle_simple_enter(dev, drv, index); else - err = tegra_cpuidle_state_enter(dev, index, cpu); + ret = tegra_cpuidle_state_enter(dev, index, cpu); - if (err && (err != -EINTR || index != TEGRA_CC6)) - pr_err_once("failed to enter state %d err: %d\n", index, err); + if (ret < 0) { + if (ret != -EINTR || index != TEGRA_CC6) + pr_err_once("failed to enter state %d err: %d\n", + index, ret); + index = -1; + } else { + index = ret; + } - return err ? -1 : index; + return index; } static int tegra114_enter_s2idle(struct cpuidle_device *dev, From 653f68b6ecd15cf147817bb01c2b7d02f4bffc8e Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 19 Aug 2020 11:23:54 +0800 Subject: [PATCH 140/265] ACPI: processor: Print more information when acpi_processor_evaluate_cst() fails Some platforms have bogus _CST which might cause unexpectd behavior in the CPU idle driver. Some bogus _CST might be unable to be disassembled by acpica-tools due to broken format. Print extra log if the _CST extraction/verification failes. This can be used to help narrow down why the CPU idle driver fails to behave as expected. Suggested-by: Zhang Rui Signed-off-by: Chen Yu [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_processor.c | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c index b51ddf3bb616..412a9725cc1e 100644 --- a/drivers/acpi/acpi_processor.c +++ b/drivers/acpi/acpi_processor.c @@ -798,22 +798,34 @@ int acpi_processor_evaluate_cst(acpi_handle handle, u32 cpu, memset(&cx, 0, sizeof(cx)); element = &cst->package.elements[i]; - if (element->type != ACPI_TYPE_PACKAGE) + if (element->type != ACPI_TYPE_PACKAGE) { + acpi_handle_info(handle, "_CST C%d type(%x) is not package, skip...\n", + i, element->type); continue; + } - if (element->package.count != 4) + if (element->package.count != 4) { + acpi_handle_info(handle, "_CST C%d package count(%d) is not 4, skip...\n", + i, element->package.count); continue; + } obj = &element->package.elements[0]; - if (obj->type != ACPI_TYPE_BUFFER) + if (obj->type != ACPI_TYPE_BUFFER) { + acpi_handle_info(handle, "_CST C%d package element[0] type(%x) is not buffer, skip...\n", + i, obj->type); continue; + } reg = (struct acpi_power_register *)obj->buffer.pointer; obj = &element->package.elements[1]; - if (obj->type != ACPI_TYPE_INTEGER) + if (obj->type != ACPI_TYPE_INTEGER) { + acpi_handle_info(handle, "_CST C[%d] package element[1] type(%x) is not integer, skip...\n", + i, obj->type); continue; + } cx.type = obj->integer.value; /* @@ -850,6 +862,8 @@ int acpi_processor_evaluate_cst(acpi_handle handle, u32 cpu, cx.entry_method = ACPI_CSTATE_HALT; snprintf(cx.desc, ACPI_CX_DESC_LEN, "ACPI HLT"); } else { + acpi_handle_info(handle, "_CST C%d declares FIXED_HARDWARE C-state but not supported in hardware, skip...\n", + i); continue; } } else if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_IO) { @@ -857,6 +871,8 @@ int acpi_processor_evaluate_cst(acpi_handle handle, u32 cpu, snprintf(cx.desc, ACPI_CX_DESC_LEN, "ACPI IOPORT 0x%x", cx.address); } else { + acpi_handle_info(handle, "_CST C%d space_id(%x) neither FIXED_HARDWARE nor SYSTEM_IO, skip...\n", + i, reg->space_id); continue; } @@ -864,14 +880,20 @@ int acpi_processor_evaluate_cst(acpi_handle handle, u32 cpu, cx.valid = 1; obj = &element->package.elements[2]; - if (obj->type != ACPI_TYPE_INTEGER) + if (obj->type != ACPI_TYPE_INTEGER) { + acpi_handle_info(handle, "_CST C%d package element[2] type(%x) not integer, skip...\n", + i, obj->type); continue; + } cx.latency = obj->integer.value; obj = &element->package.elements[3]; - if (obj->type != ACPI_TYPE_INTEGER) + if (obj->type != ACPI_TYPE_INTEGER) { + acpi_handle_info(handle, "_CST C%d package element[3] type(%x) not integer, skip...\n", + i, obj->type); continue; + } memcpy(&info->states[++last_index], &cx, sizeof(cx)); } From 10942019040c5557556ec22aae0f771b2a1a1a6d Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Tue, 1 Sep 2020 16:28:58 +0200 Subject: [PATCH 141/265] firmware: psci: Extend psci_set_osi_mode() to allow reset to PC mode The current user (cpuidle-psci) of psci_set_osi_mode() only needs to enable the PSCI OSI mode. Although, as subsequent changes shows, there is a need to be able to reset back into the PSCI PC mode. Therefore, let's extend psci_set_osi_mode() to take a bool as in-parameter, to let the user indicate whether to enable OSI or to switch back to PC mode. Reviewed-by: Sudeep Holla Signed-off-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- drivers/cpuidle/cpuidle-psci-domain.c | 2 +- drivers/firmware/psci/psci.c | 12 +++++++----- include/linux/psci.h | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c index b6e9649ab0da..b6ab0415f450 100644 --- a/drivers/cpuidle/cpuidle-psci-domain.c +++ b/drivers/cpuidle/cpuidle-psci-domain.c @@ -278,7 +278,7 @@ static int psci_cpuidle_domain_probe(struct platform_device *pdev) goto remove_pd; /* Try to enable OSI mode. */ - ret = psci_set_osi_mode(); + ret = psci_set_osi_mode(true); if (ret) { pr_warn("failed to enable OSI mode: %d\n", ret); psci_pd_remove_topology(np); diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c index 92013ecc2d9e..00af99b6f97c 100644 --- a/drivers/firmware/psci/psci.c +++ b/drivers/firmware/psci/psci.c @@ -151,12 +151,15 @@ static u32 psci_get_version(void) return invoke_psci_fn(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0); } -int psci_set_osi_mode(void) +int psci_set_osi_mode(bool enable) { + unsigned long suspend_mode; int err; - err = invoke_psci_fn(PSCI_1_0_FN_SET_SUSPEND_MODE, - PSCI_1_0_SUSPEND_MODE_OSI, 0, 0); + suspend_mode = enable ? PSCI_1_0_SUSPEND_MODE_OSI : + PSCI_1_0_SUSPEND_MODE_PC; + + err = invoke_psci_fn(PSCI_1_0_FN_SET_SUSPEND_MODE, suspend_mode, 0, 0); return psci_to_linux_errno(err); } @@ -546,8 +549,7 @@ static int __init psci_1_0_init(struct device_node *np) pr_info("OSI mode supported.\n"); /* Default to PC mode. */ - invoke_psci_fn(PSCI_1_0_FN_SET_SUSPEND_MODE, - PSCI_1_0_SUSPEND_MODE_PC, 0, 0); + psci_set_osi_mode(false); } return 0; diff --git a/include/linux/psci.h b/include/linux/psci.h index 14ad9b9ebcd6..2a1bfb890e58 100644 --- a/include/linux/psci.h +++ b/include/linux/psci.h @@ -18,7 +18,7 @@ bool psci_tos_resident_on(int cpu); int psci_cpu_suspend_enter(u32 state); bool psci_power_state_is_valid(u32 state); -int psci_set_osi_mode(void); +int psci_set_osi_mode(bool enable); bool psci_has_osi_support(void); struct psci_operations { From 70c179b49870929ca183421935415622d30875b5 Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Tue, 1 Sep 2020 16:28:59 +0200 Subject: [PATCH 142/265] cpuidle: psci: Allow PM domain to be initialized even if no OSI mode If the PSCI OSI mode isn't supported or fails to be enabled, the PM domain topology with the genpd providers isn't initialized. This is perfectly fine from cpuidle-psci point of view. However, since the PM domain topology in the DTS files is a description of the HW, no matter of whether the PSCI OSI mode is supported or not, other consumers besides the CPUs may rely on it. Therefore, let's always allow the initialization of the PM domain topology to succeed, independently of whether the PSCI OSI mode is supported. Consequentially we need to track if we succeed to enable the OSI mode, as to know when a domain idlestate can be selected. Note that, CPU devices are still not being attached to the PM domain topology, unless the PSCI OSI mode is supported. Acked-by: Sudeep Holla Signed-off-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- drivers/cpuidle/cpuidle-psci-domain.c | 59 ++++++++++++++------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c index b6ab0415f450..4a031c62f92a 100644 --- a/drivers/cpuidle/cpuidle-psci-domain.c +++ b/drivers/cpuidle/cpuidle-psci-domain.c @@ -105,7 +105,7 @@ static void psci_pd_free_states(struct genpd_power_state *states, kfree(states); } -static int psci_pd_init(struct device_node *np) +static int psci_pd_init(struct device_node *np, bool use_osi) { struct generic_pm_domain *pd; struct psci_pd_provider *pd_provider; @@ -135,11 +135,16 @@ static int psci_pd_init(struct device_node *np) pd->free_states = psci_pd_free_states; pd->name = kbasename(pd->name); - pd->power_off = psci_pd_power_off; pd->states = states; pd->state_count = state_count; pd->flags |= GENPD_FLAG_IRQ_SAFE | GENPD_FLAG_CPU_DOMAIN; + /* Allow power off when OSI has been successfully enabled. */ + if (use_osi) + pd->power_off = psci_pd_power_off; + else + pd->flags |= GENPD_FLAG_ALWAYS_ON; + /* Use governor for CPU PM domains if it has some states to manage. */ pd_gov = state_count > 0 ? &pm_domain_cpu_gov : NULL; @@ -190,7 +195,7 @@ static void psci_pd_remove(void) } } -static int psci_pd_init_topology(struct device_node *np, bool add) +static int psci_pd_init_topology(struct device_node *np) { struct device_node *node; struct of_phandle_args child, parent; @@ -203,9 +208,7 @@ static int psci_pd_init_topology(struct device_node *np, bool add) child.np = node; child.args_count = 0; - - ret = add ? of_genpd_add_subdomain(&parent, &child) : - of_genpd_remove_subdomain(&parent, &child); + ret = of_genpd_add_subdomain(&parent, &child); of_node_put(parent.np); if (ret) { of_node_put(node); @@ -216,14 +219,20 @@ static int psci_pd_init_topology(struct device_node *np, bool add) return 0; } -static int psci_pd_add_topology(struct device_node *np) +static bool psci_pd_try_set_osi_mode(void) { - return psci_pd_init_topology(np, true); -} + int ret; -static void psci_pd_remove_topology(struct device_node *np) -{ - psci_pd_init_topology(np, false); + if (!psci_has_osi_support()) + return false; + + ret = psci_set_osi_mode(true); + if (ret) { + pr_warn("failed to enable OSI mode: %d\n", ret); + return false; + } + + return true; } static void psci_cpuidle_domain_sync_state(struct device *dev) @@ -244,14 +253,14 @@ static int psci_cpuidle_domain_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct device_node *node; + bool use_osi; int ret = 0, pd_count = 0; if (!np) return -ENODEV; - /* Currently limit the hierarchical topology to be used in OSI mode. */ - if (!psci_has_osi_support()) - return 0; + /* If OSI mode is supported, let's try to enable it. */ + use_osi = psci_pd_try_set_osi_mode(); /* * Parse child nodes for the "#power-domain-cells" property and @@ -261,7 +270,7 @@ static int psci_cpuidle_domain_probe(struct platform_device *pdev) if (!of_find_property(node, "#power-domain-cells", NULL)) continue; - ret = psci_pd_init(node); + ret = psci_pd_init(node, use_osi); if (ret) goto put_node; @@ -270,30 +279,24 @@ static int psci_cpuidle_domain_probe(struct platform_device *pdev) /* Bail out if not using the hierarchical CPU topology. */ if (!pd_count) - return 0; + goto no_pd; /* Link genpd masters/subdomains to model the CPU topology. */ - ret = psci_pd_add_topology(np); + ret = psci_pd_init_topology(np); if (ret) goto remove_pd; - /* Try to enable OSI mode. */ - ret = psci_set_osi_mode(true); - if (ret) { - pr_warn("failed to enable OSI mode: %d\n", ret); - psci_pd_remove_topology(np); - goto remove_pd; - } - pr_info("Initialized CPU PM domain topology\n"); return 0; put_node: of_node_put(node); remove_pd: - if (pd_count) - psci_pd_remove(); + psci_pd_remove(); pr_err("failed to create CPU PM domains ret=%d\n", ret); +no_pd: + if (use_osi) + psci_set_osi_mode(false); return ret; } From 3ffe2e7318ba705b8e0060d4696cafc75170e819 Mon Sep 17 00:00:00 2001 From: Liu Shixin Date: Tue, 15 Sep 2020 11:26:29 +0800 Subject: [PATCH 143/265] PM: AVS: qcom-cpr: simplify the return expression of cpr_disable() Simplify the return expression. Signed-off-by: Liu Shixin Reviewed-by: Bjorn Andersson Acked-by: Kevin Hilman [ rjw: Minor subject edits ] Signed-off-by: Rafael J. Wysocki --- drivers/power/avs/qcom-cpr.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/power/avs/qcom-cpr.c b/drivers/power/avs/qcom-cpr.c index bd7c3e48b386..b24cc77d1889 100644 --- a/drivers/power/avs/qcom-cpr.c +++ b/drivers/power/avs/qcom-cpr.c @@ -665,8 +665,6 @@ static int cpr_enable(struct cpr_drv *drv) static int cpr_disable(struct cpr_drv *drv) { - int ret; - mutex_lock(&drv->lock); if (cpr_is_allowed(drv)) { @@ -676,11 +674,7 @@ static int cpr_disable(struct cpr_drv *drv) mutex_unlock(&drv->lock); - ret = regulator_disable(drv->vdd_apc); - if (ret) - return ret; - - return 0; + return regulator_disable(drv->vdd_apc); } static int cpr_config(struct cpr_drv *drv) From bf23e1cd46d6a3277aad5921e14eac40bf32b90a Mon Sep 17 00:00:00 2001 From: Kevin Hilman Date: Fri, 18 Sep 2020 13:16:33 -0700 Subject: [PATCH 144/265] MAINTAINERS: drop myself from PM AVS drivers I haven't had the time or the expertise to adequately review and maintain these drivers for awhile, so make it official. Signed-off-by: Kevin Hilman Signed-off-by: Rafael J. Wysocki --- MAINTAINERS | 1 - 1 file changed, 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index d746519253c3..4f1a56f6efaa 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5380,7 +5380,6 @@ F: include/linux/kobj* F: lib/kobj* DRIVERS FOR ADAPTIVE VOLTAGE SCALING (AVS) -M: Kevin Hilman M: Nishanth Menon L: linux-pm@vger.kernel.org S: Maintained From f49735f4978f479b0de4f50ab217d5a56bc83c55 Mon Sep 17 00:00:00 2001 From: Lina Iyer Date: Tue, 22 Sep 2020 12:34:16 -0600 Subject: [PATCH 145/265] cpuidle: record state entry rejection statistics CPUs may fail to enter the chosen idle state if there was a pending interrupt, causing the cpuidle driver to return an error value. Record that and export it via sysfs along with the other idle state statistics. This could prove useful in understanding behavior of the governor and the system during usecases that involve multiple CPUs. Signed-off-by: Lina Iyer [ rjw: Changelog and documentation edits ] Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/pm/cpuidle.rst | 9 +++++++++ drivers/cpuidle/cpuidle.c | 1 + drivers/cpuidle/sysfs.c | 3 +++ include/linux/cpuidle.h | 1 + 4 files changed, 14 insertions(+) diff --git a/Documentation/admin-guide/pm/cpuidle.rst b/Documentation/admin-guide/pm/cpuidle.rst index a96a423e3779..830868e526f4 100644 --- a/Documentation/admin-guide/pm/cpuidle.rst +++ b/Documentation/admin-guide/pm/cpuidle.rst @@ -528,6 +528,10 @@ object corresponding to it, as follows: Total number of times the hardware has been asked by the given CPU to enter this idle state. +``rejected`` + Total number of times a request to enter this idle state on the given + CPU was rejected. + The :file:`desc` and :file:`name` files both contain strings. The difference between them is that the name is expected to be more concise, while the description may be longer and it may contain white space or special characters. @@ -572,6 +576,11 @@ particular case. For these reasons, the only reliable way to find out how much time has been spent by the hardware in different idle states supported by it is to use idle state residency counters in the hardware, if available. +Generally, an interrupt received when trying to enter an idle state causes the +idle state entry request to be rejected, in which case the ``CPUIdle`` driver +may return an error code to indicate that this was the case. The :file:`usage` +and :file:`rejected` files report the number of times the given idle state +was entered successfully or rejected, respectively. .. _cpu-pm-qos: diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index 6c7e5621cf9a..0ed5030b89d6 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -307,6 +307,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, } } else { dev->last_residency_ns = 0; + dev->states_usage[index].rejected++; } return entered_state; diff --git a/drivers/cpuidle/sysfs.c b/drivers/cpuidle/sysfs.c index 091d1caceb41..53ec9585ccd4 100644 --- a/drivers/cpuidle/sysfs.c +++ b/drivers/cpuidle/sysfs.c @@ -256,6 +256,7 @@ define_show_state_time_function(exit_latency) define_show_state_time_function(target_residency) define_show_state_function(power_usage) define_show_state_ull_function(usage) +define_show_state_ull_function(rejected) define_show_state_str_function(name) define_show_state_str_function(desc) define_show_state_ull_function(above) @@ -312,6 +313,7 @@ define_one_state_ro(latency, show_state_exit_latency); define_one_state_ro(residency, show_state_target_residency); define_one_state_ro(power, show_state_power_usage); define_one_state_ro(usage, show_state_usage); +define_one_state_ro(rejected, show_state_rejected); define_one_state_ro(time, show_state_time); define_one_state_rw(disable, show_state_disable, store_state_disable); define_one_state_ro(above, show_state_above); @@ -325,6 +327,7 @@ static struct attribute *cpuidle_state_default_attrs[] = { &attr_residency.attr, &attr_power.attr, &attr_usage.attr, + &attr_rejected.attr, &attr_time.attr, &attr_disable.attr, &attr_above.attr, diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h index 6175c77bf25e..ed0da0e58e8b 100644 --- a/include/linux/cpuidle.h +++ b/include/linux/cpuidle.h @@ -38,6 +38,7 @@ struct cpuidle_state_usage { u64 time_ns; unsigned long long above; /* Number of times it's been too deep */ unsigned long long below; /* Number of times it's been too shallow */ + unsigned long long rejected; /* Number of times idle entry was rejected */ #ifdef CONFIG_SUSPEND unsigned long long s2idle_usage; unsigned long long s2idle_time; /* in US */ From 8a3decac087aa897df5af04358c2089e52e70ac4 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Tue, 18 Aug 2020 22:24:25 +0800 Subject: [PATCH 146/265] ACPI: Add out of bounds and numa_off protections to pxm_to_node() The function should check the validity of the pxm value before using it to index the pxm_to_node_map[] array. Whilst hardening this code may be good in general, the main intent here is to enable following patches that use this function to replace acpi_map_pxm_to_node() for non SRAT usecases which should return NO_NUMA_NODE for PXM entries not matching with those in SRAT. Signed-off-by: Jonathan Cameron Reviewed-by: Barry Song Reviewed-by: Hanjun Guo Signed-off-by: Rafael J. Wysocki --- drivers/acpi/numa/srat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index 15bbaab8500b..1fb486f46ee2 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -31,7 +31,7 @@ int acpi_numa __initdata; int pxm_to_node(int pxm) { - if (pxm < 0) + if (pxm < 0 || pxm >= MAX_PXM_DOMAINS || numa_off) return NUMA_NO_NODE; return pxm_to_node_map[pxm]; } From 01feba590cd610780c463aa3d498200ba4503703 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Tue, 18 Aug 2020 22:24:26 +0800 Subject: [PATCH 147/265] ACPI: Do not create new NUMA domains from ACPI static tables that are not SRAT Several ACPI static tables contain references to proximity domains. ACPI 6.3 has clarified that only entries in SRAT may define a new domain (sec 5.2.16). Those tables described in the ACPI spec have additional clarifying text. NFIT: Table 5-132, "Integer that represents the proximity domain to which the memory belongs. This number must match with corresponding entry in the SRAT table." HMAT: Table 5-145, "... This number must match with the corresponding entry in the SRAT table's processor affinity structure ... if the initiator is a processor, or the Generic Initiator Affinity Structure if the initiator is a generic initiator". IORT and DMAR are defined by external specifications. Intel Virtualization Technology for Directed I/O Rev 3.1 does not make any explicit statements, but the general SRAT statement above will still apply. https://software.intel.com/sites/default/files/managed/c5/15/vt-directed-io-spec.pdf IO Remapping Table, Platform Design Document rev D, also makes not explicit statement, but refers to ACPI SRAT table for more information and again the generic SRAT statement above applies. https://developer.arm.com/documentation/den0049/d/ In conclusion, any proximity domain specified in these tables, should be a reference to a proximity domain also found in SRAT, and they should not be able to instantiate a new domain. Hence we switch to pxm_to_node() which will only return existing nodes. Signed-off-by: Jonathan Cameron Reviewed-by: Barry Song Reviewed-by: Hanjun Guo Signed-off-by: Rafael J. Wysocki --- drivers/acpi/arm64/iort.c | 2 +- drivers/acpi/nfit/core.c | 3 +-- drivers/acpi/numa/hmat.c | 2 +- drivers/iommu/intel/dmar.c | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index ec782e4a0fe4..26005a15cb8b 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -1335,7 +1335,7 @@ static int __init arm_smmu_v3_set_proximity(struct device *dev, smmu = (struct acpi_iort_smmu_v3 *)node->node_data; if (smmu->flags & ACPI_IORT_SMMU_V3_PXM_VALID) { - int dev_node = acpi_map_pxm_to_node(smmu->pxm); + int dev_node = pxm_to_node(smmu->pxm); if (dev_node != NUMA_NO_NODE && !node_online(dev_node)) return -EINVAL; diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index 26dd208a0d63..ea0557cb54f7 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -3008,8 +3008,7 @@ static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc, if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) { ndr_desc->numa_node = acpi_map_pxm_to_online_node( spa->proximity_domain); - ndr_desc->target_node = acpi_map_pxm_to_node( - spa->proximity_domain); + ndr_desc->target_node = pxm_to_node(spa->proximity_domain); } else { ndr_desc->numa_node = NUMA_NO_NODE; ndr_desc->target_node = NUMA_NO_NODE; diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index 2c32cfb72370..cf6df2df26cd 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -666,7 +666,7 @@ static void hmat_register_target_device(struct memory_target *target, pdev->dev.numa_node = acpi_map_pxm_to_online_node(target->memory_pxm); info = (struct memregion_info) { - .target_node = acpi_map_pxm_to_node(target->memory_pxm), + .target_node = pxm_to_node(target->memory_pxm), }; rc = platform_device_add_data(pdev, &info, sizeof(info)); if (rc < 0) { diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 93e6345f3414..2f3badd41e1b 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -473,7 +473,7 @@ static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg) rhsa = (struct acpi_dmar_rhsa *)header; for_each_drhd_unit(drhd) { if (drhd->reg_base_addr == rhsa->base_address) { - int node = acpi_map_pxm_to_node(rhsa->proximity_domain); + int node = pxm_to_node(rhsa->proximity_domain); if (!node_online(node)) node = NUMA_NO_NODE; From fe205d984e7730f4d21f6f8ebc60f0698404ac31 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Tue, 18 Aug 2020 22:24:27 +0800 Subject: [PATCH 148/265] ACPI: Remove side effect of partly creating a node in acpi_map_pxm_to_online_node() While this function will only return an online node, it can have the side effect of partially creating a new node. The existing comments suggest this is intentional, but the usecases of this function are related to NFIT and HMAT parsing, neither of which should be able to define new nodes. One route by which the existing behaviour would cause a crash is to have a _PXM entry in ACPI DSDT attempt to place a device within this partly created proximity domain. A subsequent call to devm_kzalloc() or similar would result in an attempt to allocate memory on a node for which zone lists have not been set up and a NULL pointer dereference. Prevent such cases by switching to pxm_to_node() within acpi_map_pxm_to_online_node() which cannot cause a new node to be partly created. If one would previously have been created we now return NO_NUMA_NODE. Documentation updated to reflect this change. Signed-off-by: Jonathan Cameron Reviewed-by: Hanjun Guo Signed-off-by: Rafael J. Wysocki --- include/linux/acpi.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 1e4cdc6c7ae2..a9fd122ae878 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -430,13 +430,12 @@ int acpi_get_node(acpi_handle handle); * ACPI device drivers, which are called after the NUMA initialization has * completed in the kernel, can call this interface to obtain their device * NUMA topology from ACPI tables. Such drivers do not have to deal with - * offline nodes. A node may be offline when a device proximity ID is - * unique, SRAT memory entry does not exist, or NUMA is disabled, ex. - * "numa=off" on x86. + * offline nodes. A node may be offline when SRAT memory entry does not exist, + * or NUMA is disabled, ex. "numa=off" on x86. */ static inline int acpi_map_pxm_to_online_node(int pxm) { - int node = acpi_map_pxm_to_node(pxm); + int node = pxm_to_node(pxm); return numa_map_to_online_node(node); } From 4eb3723f18e9ba4d4b13d82b6f7e68dd50a852ea Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Tue, 18 Aug 2020 22:24:28 +0800 Subject: [PATCH 149/265] ACPI: Rename acpi_map_pxm_to_online_node() to pxm_to_online_node() As this function is no longer allowed to create new mappings let us rename it to reflect this. Note all nodes should already exist before any of the users of this function are called. Signed-off-by: Jonathan Cameron Reviewed-by: Hanjun Guo Signed-off-by: Rafael J. Wysocki --- drivers/acpi/nfit/core.c | 3 +-- drivers/acpi/numa/hmat.c | 2 +- include/linux/acpi.h | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index ea0557cb54f7..8df2f16d1fdf 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -3006,8 +3006,7 @@ static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc, ndr_desc->provider_data = nfit_spa; ndr_desc->attr_groups = acpi_nfit_region_attribute_groups; if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) { - ndr_desc->numa_node = acpi_map_pxm_to_online_node( - spa->proximity_domain); + ndr_desc->numa_node = pxm_to_online_node(spa->proximity_domain); ndr_desc->target_node = pxm_to_node(spa->proximity_domain); } else { ndr_desc->numa_node = NUMA_NO_NODE; diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index cf6df2df26cd..e7add2609c03 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -664,7 +664,7 @@ static void hmat_register_target_device(struct memory_target *target, goto out_pdev; } - pdev->dev.numa_node = acpi_map_pxm_to_online_node(target->memory_pxm); + pdev->dev.numa_node = pxm_to_online_node(target->memory_pxm); info = (struct memregion_info) { .target_node = pxm_to_node(target->memory_pxm), }; diff --git a/include/linux/acpi.h b/include/linux/acpi.h index a9fd122ae878..e9f6cd67943e 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -420,10 +420,10 @@ int acpi_map_pxm_to_node(int pxm); int acpi_get_node(acpi_handle handle); /** - * acpi_map_pxm_to_online_node - Map proximity ID to online node + * pxm_to_online_node - Map proximity ID to online node * @pxm: ACPI proximity ID * - * This is similar to acpi_map_pxm_to_node(), but always returns an online + * This is similar to pxm_to_node(), but always returns an online * node. When the mapped node from a given proximity ID is offline, it * looks up the node distance table and returns the nearest online node. * @@ -433,14 +433,14 @@ int acpi_get_node(acpi_handle handle); * offline nodes. A node may be offline when SRAT memory entry does not exist, * or NUMA is disabled, ex. "numa=off" on x86. */ -static inline int acpi_map_pxm_to_online_node(int pxm) +static inline int pxm_to_online_node(int pxm) { int node = pxm_to_node(pxm); return numa_map_to_online_node(node); } #else -static inline int acpi_map_pxm_to_online_node(int pxm) +static inline int pxm_to_online_node(int pxm) { return 0; } From a62d07e0006a3a3ce77041ca07f3c488ec880790 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Tue, 18 Aug 2020 22:24:29 +0800 Subject: [PATCH 150/265] ACPI: Remove side effect of partly creating a node in acpi_get_node() acpi_get_node() calls acpi_get_pxm() to evaluate the _PXM AML method for entries found in DSDT/SSDT. ACPI 6.3 sec 6.2.14 states "_PXM evaluates to an integer that identifies a device as belonging to a Proximity Domain defined in the System Resource Affinity Table (SRAT)." Hence a _PXM method should not result in creation of a new NUMA node. Before this patch, _PXM could result in partial instantiation of NUMA node, missing elements such as zone lists. A call to devm_kzalloc(), for example, results in a NULL pointer dereference. This patch therefore replaces the acpi_map_pxm_to_node() with a call to pxm_to_node(). Signed-off-by: Jonathan Cameron Reviewed-by: Hanjun Guo Reviewed-by: Barry Song Signed-off-by: Rafael J. Wysocki --- drivers/acpi/numa/srat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index 1fb486f46ee2..2c9a66c203ff 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -436,6 +436,6 @@ int acpi_get_node(acpi_handle handle) pxm = acpi_get_pxm(handle); - return acpi_map_pxm_to_node(pxm); + return pxm_to_node(pxm); } EXPORT_SYMBOL(acpi_get_node); From 95ac5bf4e471ab1f37bca522e5ea4e474fbcec31 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Tue, 18 Aug 2020 22:24:30 +0800 Subject: [PATCH 151/265] irq-chip/gic-v3-its: Fix crash if ITS is in a proximity domain without processor or memory Note this crash is present before any of the patches in this series, but as explained below it is highly unlikely anyone is shipping a firmware that causes it. Tests were done using an overriden SRAT. On ARM64, the gic-v3 driver directly parses SRAT to locate GIC Interrupt Translation Service (ITS) Affinity Structures. This is done much later in the boot than the parses of SRAT which identify proximity domains. As a result, an ITS placed in a proximity domain that is not defined by another SRAT structure will result in a NUMA node that is not completely configured and a crash. ITS [mem 0x202100000-0x20211ffff] ITS@0x0000000202100000: Using ITS number 0 Unable to handle kernel paging request at virtual address 0000000000001a08 ... Call trace: __alloc_pages_nodemask+0xe8/0x338 alloc_pages_node.constprop.0+0x34/0x40 its_probe_one+0x2f8/0xb18 gic_acpi_parse_madt_its+0x108/0x150 acpi_table_parse_entries_array+0x17c/0x264 acpi_table_parse_entries+0x48/0x6c acpi_table_parse_madt+0x30/0x3c its_init+0x1c4/0x644 gic_init_bases+0x4b8/0x4ec gic_acpi_init+0x134/0x264 acpi_match_madt+0x4c/0x84 acpi_table_parse_entries_array+0x17c/0x264 acpi_table_parse_entries+0x48/0x6c acpi_table_parse_madt+0x30/0x3c __acpi_probe_device_table+0x8c/0xe8 irqchip_init+0x3c/0x48 init_IRQ+0xcc/0x100 start_kernel+0x33c/0x548 ACPI 6.3 allows any set of Affinity Structures in SRAT to define a proximity domain. However, as we do not see this crash, we can conclude that no firmware is currently placing an ITS in a node that is separate from those containing memory and / or processors. We could modify the SRAT parsing behavior to identify the existence of Proximity Domains unique to the ITS structures, and handle them as a special case of a generic initiator (once support for those merges). This patch avoids the complexity that would be needed to handle this corner case, by not allowing the ITS entry parsing code to instantiate new NUMA Nodes. If one is encountered that does not already exist, then NO_NUMA_NODE is assigned and a warning printed just as if the value had been greater than allowed NUMA Nodes. "SRAT: Invalid NUMA node -1 in ITS affinity" Whilst this does not provide the full flexibility allowed by ACPI, it does fix the problem. We can revisit a more sophisticated solution if needed by future platforms. Change is simply to replace acpi_map_pxm_to_node with pxm_to_node reflecting the fact a new mapping is not created. Signed-off-by: Jonathan Cameron Reviewed-by: Hanjun Guo Signed-off-by: Rafael J. Wysocki --- drivers/irqchip/irq-gic-v3-its.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 548de7538632..c8ffec5d9b8a 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -5263,7 +5263,12 @@ static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header, return -EINVAL; } - node = acpi_map_pxm_to_node(its_affinity->proximity_domain); + /* + * Note that in theory a new proximity node could be created by this + * entry as it is an SRAT resource allocation structure. + * We do not currently support doing so. + */ + node = pxm_to_node(its_affinity->proximity_domain); if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) { pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node); From 749626fa099085ca77f3ff05a8efcb3b87721724 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Tue, 1 Sep 2020 14:47:16 +0800 Subject: [PATCH 152/265] ACPI: Remove three unused inline functions There is no callers in tree. Signed-off-by: YueHaibing [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- include/linux/acpi.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 1e4cdc6c7ae2..7ce2235f99f9 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -977,8 +977,6 @@ int acpi_subsys_runtime_suspend(struct device *dev); int acpi_subsys_runtime_resume(struct device *dev); int acpi_dev_pm_attach(struct device *dev, bool power_on); #else -static inline int acpi_dev_runtime_suspend(struct device *dev) { return 0; } -static inline int acpi_dev_runtime_resume(struct device *dev) { return 0; } static inline int acpi_subsys_runtime_suspend(struct device *dev) { return 0; } static inline int acpi_subsys_runtime_resume(struct device *dev) { return 0; } static inline int acpi_dev_pm_attach(struct device *dev, bool power_on) @@ -1216,13 +1214,6 @@ static inline int acpi_node_prop_get(const struct fwnode_handle *fwnode, return -ENXIO; } -static inline int acpi_dev_prop_get(const struct acpi_device *adev, - const char *propname, - void **valptr) -{ - return -ENXIO; -} - static inline int acpi_dev_prop_read_single(const struct acpi_device *adev, const char *propname, enum dev_prop_type proptype, From ad4a0f240882135bcedc207bb52e098029e5fbc7 Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Mon, 7 Sep 2020 20:57:29 +0800 Subject: [PATCH 153/265] ACPI: APD: Add kerneldoc for properties in struct apd_device_desc commit 7ff55d174cbf ("ACPI / APD: Provide build-in properties of the UART") added the struct property_entry *properties field to struct apd_device_desc, but didn't add the kernel doc for it, so add it now. Signed-off-by: Hanjun Guo [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_apd.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index 806b8ce05624..2a3db9956229 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -34,6 +34,7 @@ struct apd_private_data; * @flags: device flags like %ACPI_APD_SYSFS, %ACPI_APD_PM * @fixed_clk_rate: fixed rate input clock source for acpi device; * 0 means no fixed rate input clock source + * @properties: build-in properties of the device such as UART * @setup: a hook routine to set device resource during create platform device * * Device description defined as acpi_device_id.driver_data From 62d2234d333a89557cd398472dcf9d00ae624a8d Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Mon, 7 Sep 2020 20:57:30 +0800 Subject: [PATCH 154/265] ACPI: APD: Remove flags from struct apd_device_desc The flags field is not used in anywhere, so remove it along with two symbols related to it. Signed-off-by: Hanjun Guo [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_apd.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index 2a3db9956229..b0d17523b628 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -22,16 +22,8 @@ ACPI_MODULE_NAME("acpi_apd"); struct apd_private_data; -/** - * ACPI_APD_SYSFS : add device attributes in sysfs - * ACPI_APD_PM : attach power domain to device - */ -#define ACPI_APD_SYSFS BIT(0) -#define ACPI_APD_PM BIT(1) - /** * struct apd_device_desc - a descriptor for apd device - * @flags: device flags like %ACPI_APD_SYSFS, %ACPI_APD_PM * @fixed_clk_rate: fixed rate input clock source for acpi device; * 0 means no fixed rate input clock source * @properties: build-in properties of the device such as UART @@ -40,7 +32,6 @@ struct apd_private_data; * Device description defined as acpi_device_id.driver_data */ struct apd_device_desc { - unsigned int flags; unsigned int fixed_clk_rate; struct property_entry *properties; int (*setup)(struct apd_private_data *pdata); From 5df8e5f4f9cec84b72ee2607fdff364903d5d7ee Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Mon, 7 Sep 2020 20:57:31 +0800 Subject: [PATCH 155/265] ACPI: APD: Remove ACPI_MODULE_NAME() ACPI_MODULE_NAME() is used for ACPI debug output when using ACPI debug print functions, but ACPI debug print functions are not used in acpi_apd.c, so remove the ACPI_MODULE_NAME() from it. Signed-off-by: Hanjun Guo [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_apd.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index b0d17523b628..c985225218d3 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -19,7 +19,6 @@ #include "internal.h" -ACPI_MODULE_NAME("acpi_apd"); struct apd_private_data; /** From ee2bc5d2c4059622737c0461e60aba6c0be6793c Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Mon, 7 Sep 2020 20:57:32 +0800 Subject: [PATCH 156/265] ACPI: APD: Remove unnecessary APD_ADDR() macro stub Since APD_ADDR(desc) is only used when CONFIG_X86_AMD_PLATFORM_DEVICE or CONFIG_ARM64 is set, no need for the stub APD_ADDR(desc) definition covering the other cases, so remove it. Also update the comments for #endif. Signed-off-by: Hanjun Guo [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_apd.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index c985225218d3..44f49c660732 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -62,7 +62,6 @@ static int acpi_apd_setup(struct apd_private_data *pdata) } #ifdef CONFIG_X86_AMD_PLATFORM_DEVICE - static int misc_check_res(struct acpi_resource *ares, void *data) { struct resource res; @@ -133,7 +132,7 @@ static const struct apd_device_desc cz_uart_desc = { static const struct apd_device_desc fch_misc_desc = { .setup = fch_misc_setup, }; -#endif +#endif /* CONFIG_X86_AMD_PLATFORM_DEVICE */ #ifdef CONFIG_ARM64 static const struct apd_device_desc xgene_i2c_desc = { @@ -175,14 +174,10 @@ static const struct apd_device_desc hip08_spi_desc = { .setup = acpi_apd_setup, .fixed_clk_rate = 250000000, }; +#endif /* CONFIG_ARM64 */ + #endif -#else - -#define APD_ADDR(desc) (0UL) - -#endif /* CONFIG_X86_AMD_PLATFORM_DEVICE */ - /** * Create platform device during acpi scan attach handle. * Return value > 0 on success of creating device. From 32c6f3ffa0182d7f1a8f3e385de3fef74bc27b24 Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Mon, 7 Sep 2020 20:57:33 +0800 Subject: [PATCH 157/265] ACPI: APD: Clean up header file include statements Make the included header files appear in the alphabetical order and remove the unnecessary header file inclusions. Signed-off-by: Hanjun Guo [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_apd.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index 44f49c660732..39359ce0eb2c 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -7,15 +7,13 @@ * Wu, Jeff */ -#include -#include -#include -#include -#include #include +#include +#include #include #include -#include +#include +#include #include "internal.h" From 399e08f1f06693c6734f4a071573674369151ffd Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 11 Sep 2020 19:46:13 +0300 Subject: [PATCH 158/265] MAINTAINERS: Use my kernel.org address for Intel PMIC work Use my kernel.org address for Intel PMIC work. While here, upgrade status to maintainer of PMIC MFD devices. Signed-off-by: Andy Shevchenko Signed-off-by: Rafael J. Wysocki --- MAINTAINERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 0d0862b19ce5..30bee815858e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -405,7 +405,7 @@ F: drivers/platform/x86/i2c-multi-instantiate.c ACPI PMIC DRIVERS M: "Rafael J. Wysocki" M: Len Brown -R: Andy Shevchenko +R: Andy Shevchenko R: Mika Westerberg L: linux-acpi@vger.kernel.org S: Supported @@ -8944,7 +8944,7 @@ F: drivers/gpio/gpio-*cove.c F: drivers/gpio/gpio-msic.c INTEL PMIC MULTIFUNCTION DEVICE DRIVERS -R: Andy Shevchenko +M: Andy Shevchenko S: Maintained F: drivers/mfd/intel_msic.c F: drivers/mfd/intel_soc_pmic* From b226faab4e7890bbbccdf794e8b94276414f9058 Mon Sep 17 00:00:00 2001 From: Alex Hung Date: Sun, 13 Sep 2020 16:34:03 -0600 Subject: [PATCH 159/265] ACPI: video: use ACPI backlight for HP 635 Notebook The default backlight interface is AMD's radeon_bl0 which does not work on this system, so use the ACPI backlight interface on it instead. BugLink: https://bugs.launchpad.net/bugs/1894667 Cc: All applicable Signed-off-by: Alex Hung [ rjw: Changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/video_detect.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index 2499d7e3c710..36b62e9c8b69 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -282,6 +282,15 @@ static const struct dmi_system_id video_detect_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "530U4E/540U4E"), }, }, + /* https://bugs.launchpad.net/bugs/1894667 */ + { + .callback = video_detect_force_video, + .ident = "HP 635 Notebook", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"), + DMI_MATCH(DMI_PRODUCT_NAME, "HP 635 Notebook PC"), + }, + }, /* Non win8 machines which need native backlight nevertheless */ { From 00610935158fcc812a6425b8bda3f87c8689b805 Mon Sep 17 00:00:00 2001 From: Pujin Shi Date: Tue, 22 Sep 2020 12:46:52 +0800 Subject: [PATCH 160/265] powercap: include header to fix -Wmissing-prototypes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Include the linux/idle_inject.h header to fix W=1 build warning: drivers/powercap/idle_inject.c:152:6: warning: no previous prototype for ‘idle_inject_set_duration’ [-Wmissing-prototypes] drivers/powercap/idle_inject.c:167:6: warning: no previous prototype for ‘idle_inject_get_duration’ [-Wmissing-prototypes] drivers/powercap/idle_inject.c:179:6: warning: no previous prototype for ‘idle_inject_set_latency’ [-Wmissing-prototypes] drivers/powercap/idle_inject.c:195:5: warning: no previous prototype for ‘idle_inject_start’ [-Wmissing-prototypes] drivers/powercap/idle_inject.c:227:6: warning: no previous prototype for ‘idle_inject_stop’ [-Wmissing-prototypes] drivers/powercap/idle_inject.c:299:28: warning: no previous prototype for ‘idle_inject_register’ [-Wmissing-prototypes] drivers/powercap/idle_inject.c:345:6: warning: no previous prototype for ‘idle_inject_unregister’ [-Wmissing-prototypes] Signed-off-by: Pujin Shi Signed-off-by: Rafael J. Wysocki --- drivers/powercap/idle_inject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/powercap/idle_inject.c b/drivers/powercap/idle_inject.c index 4310901a074e..6e1a0043c411 100644 --- a/drivers/powercap/idle_inject.c +++ b/drivers/powercap/idle_inject.c @@ -43,6 +43,7 @@ #include #include #include +#include #include From d12544fb2aa9944b180c35914031a8384ab082c1 Mon Sep 17 00:00:00 2001 From: Xiang Chen Date: Tue, 22 Sep 2020 21:11:06 +0800 Subject: [PATCH 161/265] PM: runtime: Remove link state checks in rpm_get/put_supplier() To support runtime PM for hisi SAS driver (the driver is in directory drivers/scsi/hisi_sas), we add device link between scsi_device->sdev_gendev (consumer device) and hisi_hba->dev(supplier device) with flags DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE. After runtime suspended consumers and supplier, unload the dirver which causes a hung. We found that it called function device_release_driver_internal() to release the supplier device (hisi_hba->dev), as the device link was busy, it set the device link state to DL_STATE_SUPPLIER_UNBIND, and then it called device_release_driver_internal() to release the consumer device (scsi_device->sdev_gendev). Then it would try to call pm_runtime_get_sync() to resume the consumer device, but because consumer-supplier relation existed, it would try to resume the supplier first, but as the link state was already DL_STATE_SUPPLIER_UNBIND, so it skipped resuming the supplier and only resumed the consumer which hanged (it sends IOs to resume scsi_device while the SAS controller is suspended). Simple flow is as follows: device_release_driver_internal -> (supplier device) if device_links_busy -> device_links_unbind_consumers -> ... WRITE_ONCE(link->status, DL_STATE_SUPPLIER_UNBIND) device_release_driver_internal (consumer device) pm_runtime_get_sync -> (consumer device) ... __rpm_callback -> rpm_get_suppliers -> if link->state == DL_STATE_SUPPLIER_UNBIND -> skip the action of resuming the supplier ... pm_runtime_clean_up_links ... Correct suspend/resume ordering between a supplier device and its consumer devices (resume the supplier device before resuming consumer devices, and suspend consumer devices before suspending the supplier device) should be guaranteed by runtime PM, but the state checks in rpm_get_supplier() and rpm_put_supplier() break this rule, so remove them. Signed-off-by: Xiang Chen [ rjw: Subject and changelog edits ] Cc: All applicable Signed-off-by: Rafael J. Wysocki --- drivers/base/power/runtime.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 8143210a5c54..6f605f7820bb 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -291,8 +291,7 @@ static int rpm_get_suppliers(struct device *dev) device_links_read_lock_held()) { int retval; - if (!(link->flags & DL_FLAG_PM_RUNTIME) || - READ_ONCE(link->status) == DL_STATE_SUPPLIER_UNBIND) + if (!(link->flags & DL_FLAG_PM_RUNTIME)) continue; retval = pm_runtime_get_sync(link->supplier); @@ -312,8 +311,6 @@ static void rpm_put_suppliers(struct device *dev) list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, device_links_read_lock_held()) { - if (READ_ONCE(link->status) == DL_STATE_SUPPLIER_UNBIND) - continue; while (refcount_dec_not_one(&link->rpm_active)) pm_runtime_put(link->supplier); From 0ddfb1cf3b6b07c97cff16ea69931d986f9622ee Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Fri, 25 Sep 2020 08:38:26 -0500 Subject: [PATCH 162/265] x86/sev-es: Use GHCB accessor for setting the MMIO scratch buffer Use ghcb_set_sw_scratch() to set the GHCB scratch field, which will also set the corresponding bit in the GHCB valid_bitmap field to denote that sw_scratch is actually valid. Signed-off-by: Tom Lendacky Signed-off-by: Borislav Petkov Reviewed-by: Joerg Roedel Link: https://lkml.kernel.org/r/ba84deabdf44a7a880454fb351d189c6ad79d4ba.1601041106.git.thomas.lendacky@amd.com --- arch/x86/kernel/sev-es.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c index 6fcfdd32769f..4a96726fbaf8 100644 --- a/arch/x86/kernel/sev-es.c +++ b/arch/x86/kernel/sev-es.c @@ -751,7 +751,7 @@ static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, /* Can never be greater than 8 */ exit_info_2 = bytes; - ghcb->save.sw_scratch = ghcb_pa + offsetof(struct ghcb, shared_buffer); + ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer)); return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2); } From 759e56d9238ec0bb964965357f8bbb30ff2ec4b6 Mon Sep 17 00:00:00 2001 From: Tian Tao Date: Mon, 14 Sep 2020 08:44:50 +0800 Subject: [PATCH 163/265] PNP: quirks: Fix duplicate included pci.h linux/pci.h is included more than once. Remove the one that isn't necessary. Signed-off-by: Tian Tao [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/pnp/quirks.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/pnp/quirks.c b/drivers/pnp/quirks.c index de99f371d362..ac98b9919029 100644 --- a/drivers/pnp/quirks.c +++ b/drivers/pnp/quirks.c @@ -226,8 +226,6 @@ static void quirk_ad1815_mpu_resources(struct pnp_dev *dev) dev_info(&dev->dev, "made independent IRQ optional\n"); } -#include - static void quirk_system_pci_resources(struct pnp_dev *dev) { struct pci_dev *pdev = NULL; From 9a2e849fb6de471b82d19989a7944d3b7671793c Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Fri, 18 Sep 2020 17:13:28 +0800 Subject: [PATCH 164/265] ACPI: configfs: Add missing config_item_put() to fix refcount leak config_item_put() should be called in the drop_item callback, to decrement refcount for the config item. Fixes: 772bf1e2878ec ("ACPI: configfs: Unload SSDT on configfs entry removal") Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Cc: 4.13+ # 4.13+ Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_configfs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/acpi/acpi_configfs.c b/drivers/acpi/acpi_configfs.c index 88c8af455ea3..cf91f49101ea 100644 --- a/drivers/acpi/acpi_configfs.c +++ b/drivers/acpi/acpi_configfs.c @@ -228,6 +228,7 @@ static void acpi_table_drop_item(struct config_group *group, ACPI_INFO(("Host-directed Dynamic ACPI Table Unload")); acpi_unload_table(table->index); + config_item_put(cfg); } static struct configfs_group_operations acpi_table_group_ops = { From 38db63baf83a4ab89335c7c6fb210ecac74fa781 Mon Sep 17 00:00:00 2001 From: Tian Tao Date: Tue, 22 Sep 2020 10:11:14 +0800 Subject: [PATCH 165/265] PNP: ACPI: Fix missing-prototypes in acpi_pnp.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the following warnings. drivers/acpi/acpi_pnp.c:382:13: warning: no previous prototype for ‘acpi_pnp_init’ [-Wmissing-prototypes]. Signed-off-by: Tian Tao [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_pnp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/acpi/acpi_pnp.c b/drivers/acpi/acpi_pnp.c index f3039b93ff61..4ed755a963aa 100644 --- a/drivers/acpi/acpi_pnp.c +++ b/drivers/acpi/acpi_pnp.c @@ -11,6 +11,8 @@ #include #include +#include "internal.h" + static const struct acpi_device_id acpi_pnp_device_ids[] = { /* pata_isapnp */ {"PNP0600"}, /* Generic ESDI/IDE/ATA compatible hard disk controller */ From 8e8883cef6cea03fca63591624d6627a3e7a6eab Mon Sep 17 00:00:00 2001 From: Tian Tao Date: Tue, 22 Sep 2020 10:32:25 +0800 Subject: [PATCH 166/265] ACPI: PCI: update kernel-doc line comments Update kernel-doc line comments to fix warnings reported by make W=1: drivers/acpi/pci_root.c:71: warning: Function parameter or member 'handle' not described in 'acpi_is_root_bridge' Signed-off-by: Tian Tao [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/pci_root.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index f90e841c59f5..8b812803314a 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -62,7 +62,7 @@ static DEFINE_MUTEX(osc_lock); /** * acpi_is_root_bridge - determine whether an ACPI CA node is a PCI root bridge - * @handle - the ACPI CA node in question. + * @handle: the ACPI CA node in question. * * Note: we could make this API take a struct acpi_device * instead, but * for now, it's more convenient to operate on an acpi_handle. From 1eb3d0414520c8ee91b78b9f24cf30ffe71b6e5d Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:56:53 +0800 Subject: [PATCH 167/265] ACPI: cmos_rtc: Remove leftover ACPI_MODULE_NAME() ACPI_MODULE_NAME() is only needed for ACPICA debug functionality such as ACPI_DEBUG_PRINT() which is not used in acpi_cmos_rtc.c, remove it. Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_cmos_rtc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/acpi/acpi_cmos_rtc.c b/drivers/acpi/acpi_cmos_rtc.c index 33ac6cb428fe..67f1d33d15c4 100644 --- a/drivers/acpi/acpi_cmos_rtc.c +++ b/drivers/acpi/acpi_cmos_rtc.c @@ -15,8 +15,6 @@ #include "internal.h" -ACPI_MODULE_NAME("cmos rtc"); - static const struct acpi_device_id acpi_cmos_rtc_ids[] = { { "PNP0B00" }, { "PNP0B01" }, From cbaef23b2cb3098d7f59ea894af2315f38a8c2fc Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:56:54 +0800 Subject: [PATCH 168/265] ACPI: LPSS: Remove ACPI_MODULE_NAME() ACPI_MODULE_NAME() is only needed for ACPICA debug functionality such as ACPI_DEBUG_PRINT() which is not used in acpi_lpss.c, remove it. Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_lpss.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c index 5e2bfbcf526f..46e307ea0f78 100644 --- a/drivers/acpi/acpi_lpss.c +++ b/drivers/acpi/acpi_lpss.c @@ -26,8 +26,6 @@ #include "internal.h" -ACPI_MODULE_NAME("acpi_lpss"); - #ifdef CONFIG_X86_INTEL_LPSS #include From 8295d7900398f46f9518fa6c5465f8162b9ddc4e Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:56:55 +0800 Subject: [PATCH 169/265] ACPI: memhotplug: Remove leftover ACPICA debug functionality After commit 0a34764411aa ("ACPI / scan: Make memory hotplug drive use struct acpi_scan_handler"), all the ACPICA debug functionality was removed, remove the leftover ACPICA debug code. Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_memhotplug.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/drivers/acpi/acpi_memhotplug.c b/drivers/acpi/acpi_memhotplug.c index e294f44a7850..9cd987d5fee5 100644 --- a/drivers/acpi/acpi_memhotplug.c +++ b/drivers/acpi/acpi_memhotplug.c @@ -22,13 +22,6 @@ #define ACPI_MEMORY_DEVICE_HID "PNP0C80" #define ACPI_MEMORY_DEVICE_NAME "Hotplug Mem Device" -#define _COMPONENT ACPI_MEMORY_DEVICE_COMPONENT - -#undef PREFIX -#define PREFIX "ACPI:memory_hp:" - -ACPI_MODULE_NAME("acpi_memhotplug"); - static const struct acpi_device_id memory_device_ids[] = { {ACPI_MEMORY_DEVICE_HID, 0}, {"", 0}, From da5b64329d8ff599d9683479612b50b46b4d99bf Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:56:56 +0800 Subject: [PATCH 170/265] ACPI: platform: Remove ACPI_MODULE_NAME() ACPI_MODULE_NAME() is not used in the acpi_platform.c, remove it. Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_platform.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/acpi/acpi_platform.c b/drivers/acpi/acpi_platform.c index c05050f474cd..78d621290a35 100644 --- a/drivers/acpi/acpi_platform.c +++ b/drivers/acpi/acpi_platform.c @@ -19,8 +19,6 @@ #include "internal.h" -ACPI_MODULE_NAME("platform"); - static const struct acpi_device_id forbidden_id_list[] = { {"PNP0000", 0}, /* PIC */ {"PNP0100", 0}, /* Timer */ From d2c18c0db8c2f75bade2ebc483fd5dcc99da913b Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:56:57 +0800 Subject: [PATCH 171/265] ACPI: container: Remove leftover ACPICA debug functionality After commit 737f1a9f8082 ("ACPI / scan: Make container driver use struct acpi_scan_handler"), ACPICA debug print function calls were removed, so the leftover ACPICA debug functionality is useless, remove it. Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/container.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/acpi/container.c b/drivers/acpi/container.c index 9ea5f55d97e3..ccaa647ac3d4 100644 --- a/drivers/acpi/container.c +++ b/drivers/acpi/container.c @@ -14,9 +14,6 @@ #include "internal.h" -#define _COMPONENT ACPI_CONTAINER_COMPONENT -ACPI_MODULE_NAME("container"); - static const struct acpi_device_id container_device_ids[] = { {"ACPI0004", 0}, {"PNP0A05", 0}, From 6b168c56e5bd3cbccdd2d63cb4c15d56bdf4f7b8 Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:56:58 +0800 Subject: [PATCH 172/265] ACPI: custom_method: Remove dead ACPICA debug code ACPICA debug code _COMPONENT and ACPI_MODULE_NAME() is not used in custom_method.c, remove it. Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/custom_method.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/acpi/custom_method.c b/drivers/acpi/custom_method.c index b097ef209313..7b54dc95d36b 100644 --- a/drivers/acpi/custom_method.c +++ b/drivers/acpi/custom_method.c @@ -13,8 +13,6 @@ #include "internal.h" -#define _COMPONENT ACPI_SYSTEM_COMPONENT -ACPI_MODULE_NAME("custom_method"); MODULE_LICENSE("GPL"); static struct dentry *cm_dentry; From d0611c6e020982c691bf160854c0d004ef6f6c6c Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:56:59 +0800 Subject: [PATCH 173/265] ACPI: debugfs: Remove dead ACPICA debug code The _COMPONENT and ACPI_MODULE_NAME() were not used even when the debugfs.c was introduced, remove them. Signed-off-by: Hanjun Guo [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/debugfs.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/acpi/debugfs.c b/drivers/acpi/debugfs.c index d5ecea3715f8..074eb98d213e 100644 --- a/drivers/acpi/debugfs.c +++ b/drivers/acpi/debugfs.c @@ -10,9 +10,6 @@ #include "internal.h" -#define _COMPONENT ACPI_SYSTEM_COMPONENT -ACPI_MODULE_NAME("debugfs"); - struct dentry *acpi_debugfs_dir; EXPORT_SYMBOL_GPL(acpi_debugfs_dir); From 7ae57c6d14a28d6b04ed0fb5e05e08b113317628 Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:57:00 +0800 Subject: [PATCH 174/265] ACPI: dock: Remove dead ACPICA debug code The ACPICA debug ACPI_MODULE_NAME() definition is not used, remove it. Signed-off-by: Hanjun Guo [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/dock.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/acpi/dock.c b/drivers/acpi/dock.c index 9bd72c26ef46..45d4b7b69de8 100644 --- a/drivers/acpi/dock.c +++ b/drivers/acpi/dock.c @@ -20,8 +20,6 @@ #include "internal.h" -ACPI_MODULE_NAME("dock"); - static bool immediate_undock = 1; module_param(immediate_undock, bool, 0644); MODULE_PARM_DESC(immediate_undock, "1 (default) will cause the driver to " From 3ffa00e88df58f755b1a6ac20949ab09f92086af Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:57:01 +0800 Subject: [PATCH 175/265] ACPI: event: Remove leftover ACPICA debug code After commit (ff491a7334ac "netlink: change return-value logic of netlink_broadcast()"), ACPI_DEBUG_PRINT() was removed from event.c, so the ACPICA debug code is not used, remove it. Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/event.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/acpi/event.c b/drivers/acpi/event.c index 47f21599f2ab..170643927044 100644 --- a/drivers/acpi/event.c +++ b/drivers/acpi/event.c @@ -19,9 +19,6 @@ #include "internal.h" -#define _COMPONENT ACPI_SYSTEM_COMPONENT -ACPI_MODULE_NAME("event"); - /* ACPI notifier chain */ static BLOCKING_NOTIFIER_HEAD(acpi_chain_head); From be690f3ed1d87ac14411772750af98ad7d6cafab Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:57:02 +0800 Subject: [PATCH 176/265] ACPI: PCI: Remove unused ACPICA debug code The ACPICA debug code _COMPONENT and ACPI_MODULE_NAME() are not used, so can be removed. Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/pci_root.c | 2 -- drivers/acpi/pci_slot.c | 3 --- 2 files changed, 5 deletions(-) diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index f90e841c59f5..7a6abd3d10ef 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -24,8 +24,6 @@ #include "internal.h" -#define _COMPONENT ACPI_PCI_COMPONENT -ACPI_MODULE_NAME("pci_root"); #define ACPI_PCI_ROOT_CLASS "pci_bridge" #define ACPI_PCI_ROOT_DEVICE_NAME "PCI Root Bridge" static int acpi_pci_root_add(struct acpi_device *device, diff --git a/drivers/acpi/pci_slot.c b/drivers/acpi/pci_slot.c index ca2461d1bf14..d6cb2c27a23b 100644 --- a/drivers/acpi/pci_slot.c +++ b/drivers/acpi/pci_slot.c @@ -28,9 +28,6 @@ static int check_sta_before_sun; -#define _COMPONENT ACPI_PCI_COMPONENT -ACPI_MODULE_NAME("pci_slot"); - #define SLOT_NAME_SIZE 21 /* Inspired by #define in acpiphp.h */ struct acpi_pci_slot { From d93b767e8cb319b3062d2e6ef4328bee97da57f5 Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:57:03 +0800 Subject: [PATCH 177/265] ACPI: proc: Remove dead ACPICA debug code Remove the not used ACPICA debug code _COMPONENT and ACPI_MODULE_NAME() which were not used even when proc.c was introduced. Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/proc.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/acpi/proc.c b/drivers/acpi/proc.c index 7892980b3ce4..0cca7991f186 100644 --- a/drivers/acpi/proc.c +++ b/drivers/acpi/proc.c @@ -10,15 +10,11 @@ #include "sleep.h" #include "internal.h" -#define _COMPONENT ACPI_SYSTEM_COMPONENT - /* * this file provides support for: * /proc/acpi/wakeup */ -ACPI_MODULE_NAME("sleep") - static int acpi_system_wakeup_device_seq_show(struct seq_file *seq, void *offset) { From 34f98c29041ddbde910277fb92c1094adccd0c44 Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:57:04 +0800 Subject: [PATCH 178/265] ACPI: processor: Remove dead ACPICA debug code The ACPICA debug code is not used anywhere in processor_core.c and processor_thermal.c, remove it. Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/processor_core.c | 3 --- drivers/acpi/processor_thermal.c | 2 -- 2 files changed, 5 deletions(-) diff --git a/drivers/acpi/processor_core.c b/drivers/acpi/processor_core.c index f32beb7d7882..2ac48cda5b20 100644 --- a/drivers/acpi/processor_core.c +++ b/drivers/acpi/processor_core.c @@ -14,9 +14,6 @@ #include #include -#define _COMPONENT ACPI_PROCESSOR_COMPONENT -ACPI_MODULE_NAME("processor_core"); - static struct acpi_table_madt *get_madt_table(void) { static struct acpi_table_madt *madt; diff --git a/drivers/acpi/processor_thermal.c b/drivers/acpi/processor_thermal.c index 41feb88ee92d..6c7d05b37c98 100644 --- a/drivers/acpi/processor_thermal.c +++ b/drivers/acpi/processor_thermal.c @@ -20,8 +20,6 @@ #define PREFIX "ACPI: " #define ACPI_PROCESSOR_CLASS "processor" -#define _COMPONENT ACPI_PROCESSOR_COMPONENT -ACPI_MODULE_NAME("processor_thermal"); #ifdef CONFIG_CPU_FREQ From b16cd57ebc5d866c3b16664d8d19c91579b3aa88 Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:57:05 +0800 Subject: [PATCH 179/265] ACPI: tiny-power-button: Remove dead ACPICA debug code The ACPICA debug code is not used, can be removed. Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/tiny-power-button.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/acpi/tiny-power-button.c b/drivers/acpi/tiny-power-button.c index 6273d73c0b59..420e61b8eaae 100644 --- a/drivers/acpi/tiny-power-button.c +++ b/drivers/acpi/tiny-power-button.c @@ -4,7 +4,6 @@ #include #include -ACPI_MODULE_NAME("tiny-power-button"); MODULE_AUTHOR("Josh Triplett"); MODULE_DESCRIPTION("ACPI Tiny Power Button Driver"); MODULE_LICENSE("GPL"); From e0e13705edd903b71453b2f9668d06a728eb95d7 Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:57:06 +0800 Subject: [PATCH 180/265] ACPI: video: Remove leftover ACPICA debug code After commit (87521e16a7ab "acpi-video-detect: Rewrite backlight interface selection logic"), ACPI_DEBUG_PRINT() was remove, so ACPI_MODULE_NAME() and _COMPONENT are not used anymore, remove them. Signed-off-by: Hanjun Guo [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/video_detect.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index 2499d7e3c710..d2981f70a736 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -35,9 +35,6 @@ #include #include -ACPI_MODULE_NAME("video"); -#define _COMPONENT ACPI_VIDEO_COMPONENT - void acpi_video_unregister_backlight(void); static bool backlight_notifier_registered; From 77569c7533a6c089c67874ad6d8b442f5c20be10 Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Thu, 24 Sep 2020 10:57:07 +0800 Subject: [PATCH 181/265] ACPI: wakeup: Remove dead ACPICA debug code The ACPICA debug code of ACPI_SYSTEM_COMPONENT and ACPI_MODULE_NAME() is not used in wakeup.c, remove it. Signed-off-by: Hanjun Guo [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/wakeup.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/acpi/wakeup.c b/drivers/acpi/wakeup.c index 0b2e42530adf..f89dd9a99e6e 100644 --- a/drivers/acpi/wakeup.c +++ b/drivers/acpi/wakeup.c @@ -26,8 +26,6 @@ static DEFINE_MUTEX(acpi_wakeup_handler_mutex); * suspend/resume and isn't really required as this is called in S-state. At * that time, there is no device hotplug **/ -#define _COMPONENT ACPI_SYSTEM_COMPONENT -ACPI_MODULE_NAME("wakeup_devices") /** * acpi_enable_wakeup_devices - Enable wake-up device GPEs. From 7d1eb2d8f05014a4970df4aeed8c096bfb5e1dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Fri, 25 Sep 2020 09:37:47 +0000 Subject: [PATCH 182/265] ACPI: battery: include linux/power_supply.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit acpi/battery.h uses 'struct power_supply *', but fails to include/create any declaration of the type. Include linux/ power_supply.h to fix that. Signed-off-by: Barnabás PÅ‘cze [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- include/acpi/battery.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/acpi/battery.h b/include/acpi/battery.h index 5d8f5d910c82..b8d56b702c7a 100644 --- a/include/acpi/battery.h +++ b/include/acpi/battery.h @@ -2,6 +2,8 @@ #ifndef __ACPI_BATTERY_H #define __ACPI_BATTERY_H +#include + #define ACPI_BATTERY_CLASS "battery" #define ACPI_BATTERY_NOTIFY_STATUS 0x80 From a5dda201a0384e577884d9427dc60a336ced216f Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Wed, 16 Sep 2020 11:48:07 +0800 Subject: [PATCH 183/265] Drivers: hv: vmbus: Always use HV_HYP_PAGE_SIZE for gpadl Since the hypervisor always uses 4K as its page size, the size of PFNs used for gpadl should be HV_HYP_PAGE_SIZE rather than PAGE_SIZE, so adjust this accordingly as the preparation for supporting 16K/64K page size guests. No functional changes on x86, since PAGE_SIZE is always 4k (equals to HV_HYP_PAGE_SIZE). Signed-off-by: Boqun Feng Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200916034817.30282-2-boqun.feng@gmail.com Signed-off-by: Wei Liu --- drivers/hv/channel.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index 3ebda7707e46..4d0f8e5a88d6 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -22,9 +22,6 @@ #include "hyperv_vmbus.h" -#define NUM_PAGES_SPANNED(addr, len) \ -((PAGE_ALIGN(addr + len) >> PAGE_SHIFT) - (addr >> PAGE_SHIFT)) - static unsigned long virt_to_hvpfn(void *addr) { phys_addr_t paddr; @@ -35,7 +32,7 @@ static unsigned long virt_to_hvpfn(void *addr) else paddr = __pa(addr); - return paddr >> PAGE_SHIFT; + return paddr >> HV_HYP_PAGE_SHIFT; } /* @@ -330,7 +327,7 @@ static int create_gpadl_header(void *kbuffer, u32 size, int pfnsum, pfncount, pfnleft, pfncurr, pfnsize; - pagecount = size >> PAGE_SHIFT; + pagecount = size >> HV_HYP_PAGE_SHIFT; /* do we need a gpadl body msg */ pfnsize = MAX_SIZE_CHANNEL_MESSAGE - @@ -360,7 +357,7 @@ static int create_gpadl_header(void *kbuffer, u32 size, gpadl_header->range[0].byte_count = size; for (i = 0; i < pfncount; i++) gpadl_header->range[0].pfn_array[i] = virt_to_hvpfn( - kbuffer + PAGE_SIZE * i); + kbuffer + HV_HYP_PAGE_SIZE * i); *msginfo = msgheader; pfnsum = pfncount; @@ -412,7 +409,7 @@ static int create_gpadl_header(void *kbuffer, u32 size, */ for (i = 0; i < pfncurr; i++) gpadl_body->pfn[i] = virt_to_hvpfn( - kbuffer + PAGE_SIZE * (pfnsum + i)); + kbuffer + HV_HYP_PAGE_SIZE * (pfnsum + i)); /* add to msg header */ list_add_tail(&msgbody->msglistentry, @@ -441,7 +438,7 @@ static int create_gpadl_header(void *kbuffer, u32 size, gpadl_header->range[0].byte_count = size; for (i = 0; i < pagecount; i++) gpadl_header->range[0].pfn_array[i] = virt_to_hvpfn( - kbuffer + PAGE_SIZE * i); + kbuffer + HV_HYP_PAGE_SIZE * i); *msginfo = msgheader; } From edd9bbc1d934c21c537444203d50a502ad3d945f Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Wed, 16 Sep 2020 11:48:08 +0800 Subject: [PATCH 184/265] Drivers: hv: vmbus: Move __vmbus_open() Pure function movement, no functional changes. The move is made, because in a later change, __vmbus_open() will rely on some static functions afterwards, so we separate the move and the modification of __vmbus_open() in two patches to make it easy to review. Signed-off-by: Boqun Feng Reviewed-by: Wei Liu Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200916034817.30282-3-boqun.feng@gmail.com Signed-off-by: Wei Liu --- drivers/hv/channel.c | 309 ++++++++++++++++++++++--------------------- 1 file changed, 155 insertions(+), 154 deletions(-) diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index 4d0f8e5a88d6..1cbe8fc931fc 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -109,160 +109,6 @@ int vmbus_alloc_ring(struct vmbus_channel *newchannel, } EXPORT_SYMBOL_GPL(vmbus_alloc_ring); -static int __vmbus_open(struct vmbus_channel *newchannel, - void *userdata, u32 userdatalen, - void (*onchannelcallback)(void *context), void *context) -{ - struct vmbus_channel_open_channel *open_msg; - struct vmbus_channel_msginfo *open_info = NULL; - struct page *page = newchannel->ringbuffer_page; - u32 send_pages, recv_pages; - unsigned long flags; - int err; - - if (userdatalen > MAX_USER_DEFINED_BYTES) - return -EINVAL; - - send_pages = newchannel->ringbuffer_send_offset; - recv_pages = newchannel->ringbuffer_pagecount - send_pages; - - if (newchannel->state != CHANNEL_OPEN_STATE) - return -EINVAL; - - newchannel->state = CHANNEL_OPENING_STATE; - newchannel->onchannel_callback = onchannelcallback; - newchannel->channel_callback_context = context; - - err = hv_ringbuffer_init(&newchannel->outbound, page, send_pages); - if (err) - goto error_clean_ring; - - err = hv_ringbuffer_init(&newchannel->inbound, - &page[send_pages], recv_pages); - if (err) - goto error_clean_ring; - - /* Establish the gpadl for the ring buffer */ - newchannel->ringbuffer_gpadlhandle = 0; - - err = vmbus_establish_gpadl(newchannel, - page_address(newchannel->ringbuffer_page), - (send_pages + recv_pages) << PAGE_SHIFT, - &newchannel->ringbuffer_gpadlhandle); - if (err) - goto error_clean_ring; - - /* Create and init the channel open message */ - open_info = kmalloc(sizeof(*open_info) + - sizeof(struct vmbus_channel_open_channel), - GFP_KERNEL); - if (!open_info) { - err = -ENOMEM; - goto error_free_gpadl; - } - - init_completion(&open_info->waitevent); - open_info->waiting_channel = newchannel; - - open_msg = (struct vmbus_channel_open_channel *)open_info->msg; - open_msg->header.msgtype = CHANNELMSG_OPENCHANNEL; - open_msg->openid = newchannel->offermsg.child_relid; - open_msg->child_relid = newchannel->offermsg.child_relid; - open_msg->ringbuffer_gpadlhandle = newchannel->ringbuffer_gpadlhandle; - open_msg->downstream_ringbuffer_pageoffset = newchannel->ringbuffer_send_offset; - open_msg->target_vp = hv_cpu_number_to_vp_number(newchannel->target_cpu); - - if (userdatalen) - memcpy(open_msg->userdata, userdata, userdatalen); - - spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); - list_add_tail(&open_info->msglistentry, - &vmbus_connection.chn_msg_list); - spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); - - if (newchannel->rescind) { - err = -ENODEV; - goto error_free_info; - } - - err = vmbus_post_msg(open_msg, - sizeof(struct vmbus_channel_open_channel), true); - - trace_vmbus_open(open_msg, err); - - if (err != 0) - goto error_clean_msglist; - - wait_for_completion(&open_info->waitevent); - - spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); - list_del(&open_info->msglistentry); - spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); - - if (newchannel->rescind) { - err = -ENODEV; - goto error_free_info; - } - - if (open_info->response.open_result.status) { - err = -EAGAIN; - goto error_free_info; - } - - newchannel->state = CHANNEL_OPENED_STATE; - kfree(open_info); - return 0; - -error_clean_msglist: - spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); - list_del(&open_info->msglistentry); - spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); -error_free_info: - kfree(open_info); -error_free_gpadl: - vmbus_teardown_gpadl(newchannel, newchannel->ringbuffer_gpadlhandle); - newchannel->ringbuffer_gpadlhandle = 0; -error_clean_ring: - hv_ringbuffer_cleanup(&newchannel->outbound); - hv_ringbuffer_cleanup(&newchannel->inbound); - newchannel->state = CHANNEL_OPEN_STATE; - return err; -} - -/* - * vmbus_connect_ring - Open the channel but reuse ring buffer - */ -int vmbus_connect_ring(struct vmbus_channel *newchannel, - void (*onchannelcallback)(void *context), void *context) -{ - return __vmbus_open(newchannel, NULL, 0, onchannelcallback, context); -} -EXPORT_SYMBOL_GPL(vmbus_connect_ring); - -/* - * vmbus_open - Open the specified channel. - */ -int vmbus_open(struct vmbus_channel *newchannel, - u32 send_ringbuffer_size, u32 recv_ringbuffer_size, - void *userdata, u32 userdatalen, - void (*onchannelcallback)(void *context), void *context) -{ - int err; - - err = vmbus_alloc_ring(newchannel, send_ringbuffer_size, - recv_ringbuffer_size); - if (err) - return err; - - err = __vmbus_open(newchannel, userdata, userdatalen, - onchannelcallback, context); - if (err) - vmbus_free_ring(newchannel); - - return err; -} -EXPORT_SYMBOL_GPL(vmbus_open); - /* Used for Hyper-V Socket: a guest client's connect() to the host */ int vmbus_send_tl_connect_request(const guid_t *shv_guest_servie_id, const guid_t *shv_host_servie_id) @@ -556,6 +402,161 @@ cleanup: } EXPORT_SYMBOL_GPL(vmbus_establish_gpadl); +static int __vmbus_open(struct vmbus_channel *newchannel, + void *userdata, u32 userdatalen, + void (*onchannelcallback)(void *context), void *context) +{ + struct vmbus_channel_open_channel *open_msg; + struct vmbus_channel_msginfo *open_info = NULL; + struct page *page = newchannel->ringbuffer_page; + u32 send_pages, recv_pages; + unsigned long flags; + int err; + + if (userdatalen > MAX_USER_DEFINED_BYTES) + return -EINVAL; + + send_pages = newchannel->ringbuffer_send_offset; + recv_pages = newchannel->ringbuffer_pagecount - send_pages; + + if (newchannel->state != CHANNEL_OPEN_STATE) + return -EINVAL; + + newchannel->state = CHANNEL_OPENING_STATE; + newchannel->onchannel_callback = onchannelcallback; + newchannel->channel_callback_context = context; + + err = hv_ringbuffer_init(&newchannel->outbound, page, send_pages); + if (err) + goto error_clean_ring; + + err = hv_ringbuffer_init(&newchannel->inbound, + &page[send_pages], recv_pages); + if (err) + goto error_clean_ring; + + /* Establish the gpadl for the ring buffer */ + newchannel->ringbuffer_gpadlhandle = 0; + + err = vmbus_establish_gpadl(newchannel, + page_address(newchannel->ringbuffer_page), + (send_pages + recv_pages) << PAGE_SHIFT, + &newchannel->ringbuffer_gpadlhandle); + if (err) + goto error_clean_ring; + + /* Create and init the channel open message */ + open_info = kmalloc(sizeof(*open_info) + + sizeof(struct vmbus_channel_open_channel), + GFP_KERNEL); + if (!open_info) { + err = -ENOMEM; + goto error_free_gpadl; + } + + init_completion(&open_info->waitevent); + open_info->waiting_channel = newchannel; + + open_msg = (struct vmbus_channel_open_channel *)open_info->msg; + open_msg->header.msgtype = CHANNELMSG_OPENCHANNEL; + open_msg->openid = newchannel->offermsg.child_relid; + open_msg->child_relid = newchannel->offermsg.child_relid; + open_msg->ringbuffer_gpadlhandle = newchannel->ringbuffer_gpadlhandle; + open_msg->downstream_ringbuffer_pageoffset = newchannel->ringbuffer_send_offset; + open_msg->target_vp = hv_cpu_number_to_vp_number(newchannel->target_cpu); + + if (userdatalen) + memcpy(open_msg->userdata, userdata, userdatalen); + + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_add_tail(&open_info->msglistentry, + &vmbus_connection.chn_msg_list); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + if (newchannel->rescind) { + err = -ENODEV; + goto error_free_info; + } + + err = vmbus_post_msg(open_msg, + sizeof(struct vmbus_channel_open_channel), true); + + trace_vmbus_open(open_msg, err); + + if (err != 0) + goto error_clean_msglist; + + wait_for_completion(&open_info->waitevent); + + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_del(&open_info->msglistentry); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + if (newchannel->rescind) { + err = -ENODEV; + goto error_free_info; + } + + if (open_info->response.open_result.status) { + err = -EAGAIN; + goto error_free_info; + } + + newchannel->state = CHANNEL_OPENED_STATE; + kfree(open_info); + return 0; + +error_clean_msglist: + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_del(&open_info->msglistentry); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); +error_free_info: + kfree(open_info); +error_free_gpadl: + vmbus_teardown_gpadl(newchannel, newchannel->ringbuffer_gpadlhandle); + newchannel->ringbuffer_gpadlhandle = 0; +error_clean_ring: + hv_ringbuffer_cleanup(&newchannel->outbound); + hv_ringbuffer_cleanup(&newchannel->inbound); + newchannel->state = CHANNEL_OPEN_STATE; + return err; +} + +/* + * vmbus_connect_ring - Open the channel but reuse ring buffer + */ +int vmbus_connect_ring(struct vmbus_channel *newchannel, + void (*onchannelcallback)(void *context), void *context) +{ + return __vmbus_open(newchannel, NULL, 0, onchannelcallback, context); +} +EXPORT_SYMBOL_GPL(vmbus_connect_ring); + +/* + * vmbus_open - Open the specified channel. + */ +int vmbus_open(struct vmbus_channel *newchannel, + u32 send_ringbuffer_size, u32 recv_ringbuffer_size, + void *userdata, u32 userdatalen, + void (*onchannelcallback)(void *context), void *context) +{ + int err; + + err = vmbus_alloc_ring(newchannel, send_ringbuffer_size, + recv_ringbuffer_size); + if (err) + return err; + + err = __vmbus_open(newchannel, userdata, userdatalen, + onchannelcallback, context); + if (err) + vmbus_free_ring(newchannel); + + return err; +} +EXPORT_SYMBOL_GPL(vmbus_open); + + /* * vmbus_teardown_gpadl -Teardown the specified GPADL handle */ From c1135c7fd0e95740cd4619ce389f43ffce043140 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Wed, 16 Sep 2020 11:48:09 +0800 Subject: [PATCH 185/265] Drivers: hv: vmbus: Introduce types of GPADL This patch introduces two types of GPADL: HV_GPADL_{BUFFER, RING}. The types of GPADL are purely the concept in the guest, IOW the hypervisor treat them as the same. The reason of introducing the types for GPADL is to support guests whose page size is not 4k (the page size of Hyper-V hypervisor). In these guests, both the headers and the data parts of the ringbuffers need to be aligned to the PAGE_SIZE, because 1) some of the ringbuffers will be mapped into userspace and 2) we use "double mapping" mechanism to support fast wrap-around, and "double mapping" relies on ringbuffers being page-aligned. However, the Hyper-V hypervisor only uses 4k (HV_HYP_PAGE_SIZE) headers. Our solution to this is that we always make the headers of ringbuffers take one guest page and when GPADL is established between the guest and hypervisor, the only first 4k of header is used. To handle this special case, we need the types of GPADL to differ different guest memory usage for GPADL. Type enum is introduced along with several general interfaces to describe the differences between normal buffer GPADL and ringbuffer GPADL. Signed-off-by: Boqun Feng Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200916034817.30282-4-boqun.feng@gmail.com Signed-off-by: Wei Liu --- drivers/hv/channel.c | 160 +++++++++++++++++++++++++++++++++++------ include/linux/hyperv.h | 48 ++++++++++++- 2 files changed, 187 insertions(+), 21 deletions(-) diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index 1cbe8fc931fc..45267b6d069e 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -35,6 +35,99 @@ static unsigned long virt_to_hvpfn(void *addr) return paddr >> HV_HYP_PAGE_SHIFT; } +/* + * hv_gpadl_size - Return the real size of a gpadl, the size that Hyper-V uses + * + * For BUFFER gpadl, Hyper-V uses the exact same size as the guest does. + * + * For RING gpadl, in each ring, the guest uses one PAGE_SIZE as the header + * (because of the alignment requirement), however, the hypervisor only + * uses the first HV_HYP_PAGE_SIZE as the header, therefore leaving a + * (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap. And since there are two rings in a + * ringbuffer, the total size for a RING gpadl that Hyper-V uses is the + * total size that the guest uses minus twice of the gap size. + */ +static inline u32 hv_gpadl_size(enum hv_gpadl_type type, u32 size) +{ + switch (type) { + case HV_GPADL_BUFFER: + return size; + case HV_GPADL_RING: + /* The size of a ringbuffer must be page-aligned */ + BUG_ON(size % PAGE_SIZE); + /* + * Two things to notice here: + * 1) We're processing two ring buffers as a unit + * 2) We're skipping any space larger than HV_HYP_PAGE_SIZE in + * the first guest-size page of each of the two ring buffers. + * So we effectively subtract out two guest-size pages, and add + * back two Hyper-V size pages. + */ + return size - 2 * (PAGE_SIZE - HV_HYP_PAGE_SIZE); + } + BUG(); + return 0; +} + +/* + * hv_ring_gpadl_send_hvpgoffset - Calculate the send offset (in unit of + * HV_HYP_PAGE) in a ring gpadl based on the + * offset in the guest + * + * @offset: the offset (in bytes) where the send ringbuffer starts in the + * virtual address space of the guest + */ +static inline u32 hv_ring_gpadl_send_hvpgoffset(u32 offset) +{ + + /* + * For RING gpadl, in each ring, the guest uses one PAGE_SIZE as the + * header (because of the alignment requirement), however, the + * hypervisor only uses the first HV_HYP_PAGE_SIZE as the header, + * therefore leaving a (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap. + * + * And to calculate the effective send offset in gpadl, we need to + * substract this gap. + */ + return (offset - (PAGE_SIZE - HV_HYP_PAGE_SIZE)) >> HV_HYP_PAGE_SHIFT; +} + +/* + * hv_gpadl_hvpfn - Return the Hyper-V page PFN of the @i th Hyper-V page in + * the gpadl + * + * @type: the type of the gpadl + * @kbuffer: the pointer to the gpadl in the guest + * @size: the total size (in bytes) of the gpadl + * @send_offset: the offset (in bytes) where the send ringbuffer starts in the + * virtual address space of the guest + * @i: the index + */ +static inline u64 hv_gpadl_hvpfn(enum hv_gpadl_type type, void *kbuffer, + u32 size, u32 send_offset, int i) +{ + int send_idx = hv_ring_gpadl_send_hvpgoffset(send_offset); + unsigned long delta = 0UL; + + switch (type) { + case HV_GPADL_BUFFER: + break; + case HV_GPADL_RING: + if (i == 0) + delta = 0; + else if (i <= send_idx) + delta = PAGE_SIZE - HV_HYP_PAGE_SIZE; + else + delta = 2 * (PAGE_SIZE - HV_HYP_PAGE_SIZE); + break; + default: + BUG(); + break; + } + + return virt_to_hvpfn(kbuffer + delta + (HV_HYP_PAGE_SIZE * i)); +} + /* * vmbus_setevent- Trigger an event notification on the specified * channel. @@ -160,7 +253,8 @@ EXPORT_SYMBOL_GPL(vmbus_send_modifychannel); /* * create_gpadl_header - Creates a gpadl for the specified buffer */ -static int create_gpadl_header(void *kbuffer, u32 size, +static int create_gpadl_header(enum hv_gpadl_type type, void *kbuffer, + u32 size, u32 send_offset, struct vmbus_channel_msginfo **msginfo) { int i; @@ -173,7 +267,7 @@ static int create_gpadl_header(void *kbuffer, u32 size, int pfnsum, pfncount, pfnleft, pfncurr, pfnsize; - pagecount = size >> HV_HYP_PAGE_SHIFT; + pagecount = hv_gpadl_size(type, size) >> HV_HYP_PAGE_SHIFT; /* do we need a gpadl body msg */ pfnsize = MAX_SIZE_CHANNEL_MESSAGE - @@ -200,10 +294,10 @@ static int create_gpadl_header(void *kbuffer, u32 size, gpadl_header->range_buflen = sizeof(struct gpa_range) + pagecount * sizeof(u64); gpadl_header->range[0].byte_offset = 0; - gpadl_header->range[0].byte_count = size; + gpadl_header->range[0].byte_count = hv_gpadl_size(type, size); for (i = 0; i < pfncount; i++) - gpadl_header->range[0].pfn_array[i] = virt_to_hvpfn( - kbuffer + HV_HYP_PAGE_SIZE * i); + gpadl_header->range[0].pfn_array[i] = hv_gpadl_hvpfn( + type, kbuffer, size, send_offset, i); *msginfo = msgheader; pfnsum = pfncount; @@ -254,8 +348,8 @@ static int create_gpadl_header(void *kbuffer, u32 size, * so the hypervisor guarantees that this is ok. */ for (i = 0; i < pfncurr; i++) - gpadl_body->pfn[i] = virt_to_hvpfn( - kbuffer + HV_HYP_PAGE_SIZE * (pfnsum + i)); + gpadl_body->pfn[i] = hv_gpadl_hvpfn(type, + kbuffer, size, send_offset, pfnsum + i); /* add to msg header */ list_add_tail(&msgbody->msglistentry, @@ -281,10 +375,10 @@ static int create_gpadl_header(void *kbuffer, u32 size, gpadl_header->range_buflen = sizeof(struct gpa_range) + pagecount * sizeof(u64); gpadl_header->range[0].byte_offset = 0; - gpadl_header->range[0].byte_count = size; + gpadl_header->range[0].byte_count = hv_gpadl_size(type, size); for (i = 0; i < pagecount; i++) - gpadl_header->range[0].pfn_array[i] = virt_to_hvpfn( - kbuffer + HV_HYP_PAGE_SIZE * i); + gpadl_header->range[0].pfn_array[i] = hv_gpadl_hvpfn( + type, kbuffer, size, send_offset, i); *msginfo = msgheader; } @@ -297,15 +391,20 @@ nomem: } /* - * vmbus_establish_gpadl - Establish a GPADL for the specified buffer + * __vmbus_establish_gpadl - Establish a GPADL for a buffer or ringbuffer * * @channel: a channel + * @type: the type of the corresponding GPADL, only meaningful for the guest. * @kbuffer: from kmalloc or vmalloc * @size: page-size multiple + * @send_offset: the offset (in bytes) where the send ring buffer starts, + * should be 0 for BUFFER type gpadl * @gpadl_handle: some funky thing */ -int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer, - u32 size, u32 *gpadl_handle) +static int __vmbus_establish_gpadl(struct vmbus_channel *channel, + enum hv_gpadl_type type, void *kbuffer, + u32 size, u32 send_offset, + u32 *gpadl_handle) { struct vmbus_channel_gpadl_header *gpadlmsg; struct vmbus_channel_gpadl_body *gpadl_body; @@ -319,7 +418,7 @@ int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer, next_gpadl_handle = (atomic_inc_return(&vmbus_connection.next_gpadl_handle) - 1); - ret = create_gpadl_header(kbuffer, size, &msginfo); + ret = create_gpadl_header(type, kbuffer, size, send_offset, &msginfo); if (ret) return ret; @@ -400,6 +499,21 @@ cleanup: kfree(msginfo); return ret; } + +/* + * vmbus_establish_gpadl - Establish a GPADL for the specified buffer + * + * @channel: a channel + * @kbuffer: from kmalloc or vmalloc + * @size: page-size multiple + * @gpadl_handle: some funky thing + */ +int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer, + u32 size, u32 *gpadl_handle) +{ + return __vmbus_establish_gpadl(channel, HV_GPADL_BUFFER, kbuffer, size, + 0U, gpadl_handle); +} EXPORT_SYMBOL_GPL(vmbus_establish_gpadl); static int __vmbus_open(struct vmbus_channel *newchannel, @@ -438,10 +552,11 @@ static int __vmbus_open(struct vmbus_channel *newchannel, /* Establish the gpadl for the ring buffer */ newchannel->ringbuffer_gpadlhandle = 0; - err = vmbus_establish_gpadl(newchannel, - page_address(newchannel->ringbuffer_page), - (send_pages + recv_pages) << PAGE_SHIFT, - &newchannel->ringbuffer_gpadlhandle); + err = __vmbus_establish_gpadl(newchannel, HV_GPADL_RING, + page_address(newchannel->ringbuffer_page), + (send_pages + recv_pages) << PAGE_SHIFT, + newchannel->ringbuffer_send_offset << PAGE_SHIFT, + &newchannel->ringbuffer_gpadlhandle); if (err) goto error_clean_ring; @@ -462,7 +577,13 @@ static int __vmbus_open(struct vmbus_channel *newchannel, open_msg->openid = newchannel->offermsg.child_relid; open_msg->child_relid = newchannel->offermsg.child_relid; open_msg->ringbuffer_gpadlhandle = newchannel->ringbuffer_gpadlhandle; - open_msg->downstream_ringbuffer_pageoffset = newchannel->ringbuffer_send_offset; + /* + * The unit of ->downstream_ringbuffer_pageoffset is HV_HYP_PAGE and + * the unit of ->ringbuffer_send_offset (i.e. send_pages) is PAGE, so + * here we calculate it into HV_HYP_PAGE. + */ + open_msg->downstream_ringbuffer_pageoffset = + hv_ring_gpadl_send_hvpgoffset(send_pages << PAGE_SHIFT); open_msg->target_vp = hv_cpu_number_to_vp_number(newchannel->target_cpu); if (userdatalen) @@ -556,7 +677,6 @@ int vmbus_open(struct vmbus_channel *newchannel, } EXPORT_SYMBOL_GPL(vmbus_open); - /* * vmbus_teardown_gpadl -Teardown the specified GPADL handle */ diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 38100e80360a..9c19149c0e1a 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -29,6 +29,48 @@ #pragma pack(push, 1) +/* + * Types for GPADL, decides is how GPADL header is created. + * + * It doesn't make much difference between BUFFER and RING if PAGE_SIZE is the + * same as HV_HYP_PAGE_SIZE. + * + * If PAGE_SIZE is bigger than HV_HYP_PAGE_SIZE, the headers of ring buffers + * will be of PAGE_SIZE, however, only the first HV_HYP_PAGE will be put + * into gpadl, therefore the number for HV_HYP_PAGE and the indexes of each + * HV_HYP_PAGE will be different between different types of GPADL, for example + * if PAGE_SIZE is 64K: + * + * BUFFER: + * + * gva: |-- 64k --|-- 64k --| ... | + * gpa: | 4k | 4k | ... | 4k | 4k | 4k | ... | 4k | + * index: 0 1 2 15 16 17 18 .. 31 32 ... + * | | ... | | | ... | ... + * v V V V V V + * gpadl: | 4k | 4k | ... | 4k | 4k | 4k | ... | 4k | ... | + * index: 0 1 2 ... 15 16 17 18 .. 31 32 ... + * + * RING: + * + * | header | data | header | data | + * gva: |-- 64k --|-- 64k --| ... |-- 64k --|-- 64k --| ... | + * gpa: | 4k | .. | 4k | 4k | ... | 4k | ... | 4k | .. | 4k | .. | ... | + * index: 0 1 16 17 18 31 ... n n+1 n+16 ... 2n + * | / / / | / / + * | / / / | / / + * | / / ... / ... | / ... / + * | / / / | / / + * | / / / | / / + * V V V V V V v + * gpadl: | 4k | 4k | ... | ... | 4k | 4k | ... | + * index: 0 1 2 ... 16 ... n-15 n-14 n-13 ... 2n-30 + */ +enum hv_gpadl_type { + HV_GPADL_BUFFER, + HV_GPADL_RING +}; + /* Single-page buffer */ struct hv_page_buffer { u32 len; @@ -111,7 +153,7 @@ struct hv_ring_buffer { } feature_bits; /* Pad it to PAGE_SIZE so that data starts on page boundary */ - u8 reserved2[4028]; + u8 reserved2[PAGE_SIZE - 68]; /* * Ring data starts here + RingDataStartOffset @@ -120,6 +162,10 @@ struct hv_ring_buffer { u8 buffer[]; } __packed; +/* Calculate the proper size of a ringbuffer, it must be page-aligned */ +#define VMBUS_RING_SIZE(payload_sz) PAGE_ALIGN(sizeof(struct hv_ring_buffer) + \ + (payload_sz)) + struct hv_ring_buffer_info { struct hv_ring_buffer *ring_buffer; u32 ring_size; /* Include the shared header */ From ef514d3e38edf36764e61fc11366410041598ad9 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Wed, 16 Sep 2020 11:48:10 +0800 Subject: [PATCH 186/265] Drivers: hv: Use HV_HYP_PAGE in hv_synic_enable_regs() Both the base_*_gpa should use the guest page number in Hyper-V page, so use HV_HYP_PAGE instead of PAGE. Signed-off-by: Boqun Feng Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200916034817.30282-5-boqun.feng@gmail.com Signed-off-by: Wei Liu --- drivers/hv/hv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c index da69338f92f5..410f1fab519c 100644 --- a/drivers/hv/hv.c +++ b/drivers/hv/hv.c @@ -165,7 +165,7 @@ void hv_synic_enable_regs(unsigned int cpu) hv_get_simp(simp.as_uint64); simp.simp_enabled = 1; simp.base_simp_gpa = virt_to_phys(hv_cpu->synic_message_page) - >> PAGE_SHIFT; + >> HV_HYP_PAGE_SHIFT; hv_set_simp(simp.as_uint64); @@ -173,7 +173,7 @@ void hv_synic_enable_regs(unsigned int cpu) hv_get_siefp(siefp.as_uint64); siefp.siefp_enabled = 1; siefp.base_siefp_gpa = virt_to_phys(hv_cpu->synic_event_page) - >> PAGE_SHIFT; + >> HV_HYP_PAGE_SHIFT; hv_set_siefp(siefp.as_uint64); From bca6b91dfdd4347d8fe0bb97377dda0cacd51d82 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Wed, 16 Sep 2020 11:48:11 +0800 Subject: [PATCH 187/265] Drivers: hv: vmbus: Move virt_to_hvpfn() to hyperv header There will be more places other than vmbus where we need to calculate the Hyper-V page PFN from a virtual address, so move virt_to_hvpfn() to hyperv generic header. Signed-off-by: Boqun Feng Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200916034817.30282-6-boqun.feng@gmail.com Signed-off-by: Wei Liu --- drivers/hv/channel.c | 13 ------------- include/linux/hyperv.h | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index 45267b6d069e..fbdda9938039 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -22,19 +22,6 @@ #include "hyperv_vmbus.h" -static unsigned long virt_to_hvpfn(void *addr) -{ - phys_addr_t paddr; - - if (is_vmalloc_addr(addr)) - paddr = page_to_phys(vmalloc_to_page(addr)) + - offset_in_page(addr); - else - paddr = __pa(addr); - - return paddr >> HV_HYP_PAGE_SHIFT; -} - /* * hv_gpadl_size - Return the real size of a gpadl, the size that Hyper-V uses * diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 9c19149c0e1a..83456dc181a8 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -14,6 +14,7 @@ #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #define MAX_PAGE_BUFFER_COUNT 32 #define MAX_MULTIPAGE_BUFFER_COUNT 32 /* 128K */ @@ -1676,4 +1678,17 @@ struct hyperv_pci_block_ops { extern struct hyperv_pci_block_ops hvpci_block_ops; +static inline unsigned long virt_to_hvpfn(void *addr) +{ + phys_addr_t paddr; + + if (is_vmalloc_addr(addr)) + paddr = page_to_phys(vmalloc_to_page(addr)) + + offset_in_page(addr); + else + paddr = __pa(addr); + + return paddr >> HV_HYP_PAGE_SHIFT; +} + #endif /* _HYPERV_H */ From 106dee08408118c5b50387817e74419436e8b651 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Wed, 16 Sep 2020 11:48:12 +0800 Subject: [PATCH 188/265] hv: hyperv.h: Introduce some hvpfn helper functions When a guest communicate with the hypervisor, it must use HV_HYP_PAGE to calculate PFN, so introduce a few hvpfn helper functions as the counterpart of the page helper functions. This is the preparation for supporting guest whose PAGE_SIZE is not 4k. Signed-off-by: Boqun Feng Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200916034817.30282-7-boqun.feng@gmail.com Signed-off-by: Wei Liu --- include/linux/hyperv.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 83456dc181a8..1ce131f29f3b 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -1691,4 +1691,9 @@ static inline unsigned long virt_to_hvpfn(void *addr) return paddr >> HV_HYP_PAGE_SHIFT; } +#define NR_HV_HYP_PAGES_IN_PAGE (PAGE_SIZE / HV_HYP_PAGE_SIZE) +#define offset_in_hvpage(ptr) ((unsigned long)(ptr) & ~HV_HYP_PAGE_MASK) +#define HVPFN_UP(x) (((x) + HV_HYP_PAGE_SIZE-1) >> HV_HYP_PAGE_SHIFT) +#define page_to_hvpfn(page) (page_to_pfn(page) * NR_HV_HYP_PAGES_IN_PAGE) + #endif /* _HYPERV_H */ From 11d8620e0867cb21313aa66ff6ce0172e28f68ff Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Wed, 16 Sep 2020 11:48:13 +0800 Subject: [PATCH 189/265] hv_netvsc: Use HV_HYP_PAGE_SIZE for Hyper-V communication When communicating with Hyper-V, HV_HYP_PAGE_SIZE should be used since that's the page size used by Hyper-V and Hyper-V expects all page-related data using the unit of HY_HYP_PAGE_SIZE, for example, the "pfn" in hv_page_buffer is actually the HV_HYP_PAGE (i.e. the Hyper-V page) number. In order to support guest whose page size is not 4k, we need to make hv_netvsc always use HV_HYP_PAGE_SIZE for Hyper-V communication. Signed-off-by: Boqun Feng Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200916034817.30282-8-boqun.feng@gmail.com Signed-off-by: Wei Liu --- drivers/net/hyperv/netvsc.c | 2 +- drivers/net/hyperv/netvsc_drv.c | 46 +++++++++++++++---------------- drivers/net/hyperv/rndis_filter.c | 13 ++++----- 3 files changed, 30 insertions(+), 31 deletions(-) diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c index 41f5cf0bb997..1d6f2256da6b 100644 --- a/drivers/net/hyperv/netvsc.c +++ b/drivers/net/hyperv/netvsc.c @@ -794,7 +794,7 @@ static void netvsc_copy_to_send_buf(struct netvsc_device *net_device, } for (i = 0; i < page_count; i++) { - char *src = phys_to_virt(pb[i].pfn << PAGE_SHIFT); + char *src = phys_to_virt(pb[i].pfn << HV_HYP_PAGE_SHIFT); u32 offset = pb[i].offset; u32 len = pb[i].len; diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c index 787f17e2a971..70d91e9b4395 100644 --- a/drivers/net/hyperv/netvsc_drv.c +++ b/drivers/net/hyperv/netvsc_drv.c @@ -373,32 +373,29 @@ static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb, return txq; } -static u32 fill_pg_buf(struct page *page, u32 offset, u32 len, +static u32 fill_pg_buf(unsigned long hvpfn, u32 offset, u32 len, struct hv_page_buffer *pb) { int j = 0; - /* Deal with compound pages by ignoring unused part - * of the page. - */ - page += (offset >> PAGE_SHIFT); - offset &= ~PAGE_MASK; + hvpfn += offset >> HV_HYP_PAGE_SHIFT; + offset = offset & ~HV_HYP_PAGE_MASK; while (len > 0) { unsigned long bytes; - bytes = PAGE_SIZE - offset; + bytes = HV_HYP_PAGE_SIZE - offset; if (bytes > len) bytes = len; - pb[j].pfn = page_to_pfn(page); + pb[j].pfn = hvpfn; pb[j].offset = offset; pb[j].len = bytes; offset += bytes; len -= bytes; - if (offset == PAGE_SIZE && len) { - page++; + if (offset == HV_HYP_PAGE_SIZE && len) { + hvpfn++; offset = 0; j++; } @@ -421,23 +418,26 @@ static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb, * 2. skb linear data * 3. skb fragment data */ - slots_used += fill_pg_buf(virt_to_page(hdr), - offset_in_page(hdr), - len, &pb[slots_used]); + slots_used += fill_pg_buf(virt_to_hvpfn(hdr), + offset_in_hvpage(hdr), + len, + &pb[slots_used]); packet->rmsg_size = len; packet->rmsg_pgcnt = slots_used; - slots_used += fill_pg_buf(virt_to_page(data), - offset_in_page(data), - skb_headlen(skb), &pb[slots_used]); + slots_used += fill_pg_buf(virt_to_hvpfn(data), + offset_in_hvpage(data), + skb_headlen(skb), + &pb[slots_used]); for (i = 0; i < frags; i++) { skb_frag_t *frag = skb_shinfo(skb)->frags + i; - slots_used += fill_pg_buf(skb_frag_page(frag), - skb_frag_off(frag), - skb_frag_size(frag), &pb[slots_used]); + slots_used += fill_pg_buf(page_to_hvpfn(skb_frag_page(frag)), + skb_frag_off(frag), + skb_frag_size(frag), + &pb[slots_used]); } return slots_used; } @@ -453,8 +453,8 @@ static int count_skb_frag_slots(struct sk_buff *skb) unsigned long offset = skb_frag_off(frag); /* Skip unused frames from start of page */ - offset &= ~PAGE_MASK; - pages += PFN_UP(offset + size); + offset &= ~HV_HYP_PAGE_MASK; + pages += HVPFN_UP(offset + size); } return pages; } @@ -462,12 +462,12 @@ static int count_skb_frag_slots(struct sk_buff *skb) static int netvsc_get_slots(struct sk_buff *skb) { char *data = skb->data; - unsigned int offset = offset_in_page(data); + unsigned int offset = offset_in_hvpage(data); unsigned int len = skb_headlen(skb); int slots; int frag_slots; - slots = DIV_ROUND_UP(offset + len, PAGE_SIZE); + slots = DIV_ROUND_UP(offset + len, HV_HYP_PAGE_SIZE); frag_slots = count_skb_frag_slots(skb); return slots + frag_slots; } diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c index b81ceba38218..1e2de8fb7fec 100644 --- a/drivers/net/hyperv/rndis_filter.c +++ b/drivers/net/hyperv/rndis_filter.c @@ -25,7 +25,7 @@ static void rndis_set_multicast(struct work_struct *w); -#define RNDIS_EXT_LEN PAGE_SIZE +#define RNDIS_EXT_LEN HV_HYP_PAGE_SIZE struct rndis_request { struct list_head list_ent; struct completion wait_event; @@ -215,18 +215,17 @@ static int rndis_filter_send_request(struct rndis_device *dev, packet->page_buf_cnt = 1; pb[0].pfn = virt_to_phys(&req->request_msg) >> - PAGE_SHIFT; + HV_HYP_PAGE_SHIFT; pb[0].len = req->request_msg.msg_len; - pb[0].offset = - (unsigned long)&req->request_msg & (PAGE_SIZE - 1); + pb[0].offset = offset_in_hvpage(&req->request_msg); /* Add one page_buf when request_msg crossing page boundary */ - if (pb[0].offset + pb[0].len > PAGE_SIZE) { + if (pb[0].offset + pb[0].len > HV_HYP_PAGE_SIZE) { packet->page_buf_cnt++; - pb[0].len = PAGE_SIZE - + pb[0].len = HV_HYP_PAGE_SIZE - pb[0].offset; pb[1].pfn = virt_to_phys((void *)&req->request_msg - + pb[0].len) >> PAGE_SHIFT; + + pb[0].len) >> HV_HYP_PAGE_SHIFT; pb[1].offset = 0; pb[1].len = req->request_msg.msg_len - pb[0].len; From 50597970aa84cc087424ef8d3bc430665aebcece Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Wed, 16 Sep 2020 11:48:14 +0800 Subject: [PATCH 190/265] Input: hyperv-keyboard: Use VMBUS_RING_SIZE() for ringbuffer sizes For a Hyper-V vmbus, the size of the ringbuffer has two requirements: 1) it has to take one PAGE_SIZE for the header 2) it has to be PAGE_SIZE aligned so that double-mapping can work VMBUS_RING_SIZE() could calculate a correct ringbuffer size which fulfills both requirements, therefore use it to make sure vmbus work when PAGE_SIZE != HV_HYP_PAGE_SIZE (4K). Note that since the argument for VMBUS_RING_SIZE() is the size of payload (data part), so it will be minus 4k (the size of header when PAGE_SIZE = 4k) than the original value to keep the ringbuffer total size unchanged when PAGE_SIZE = 4k. Signed-off-by: Boqun Feng Cc: Michael Kelley Cc: Dmitry Torokhov Acked-by: Dmitry Torokhov Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200916034817.30282-9-boqun.feng@gmail.com Signed-off-by: Wei Liu --- drivers/input/serio/hyperv-keyboard.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/input/serio/hyperv-keyboard.c b/drivers/input/serio/hyperv-keyboard.c index df4e9f6f4529..1a7b72a9016d 100644 --- a/drivers/input/serio/hyperv-keyboard.c +++ b/drivers/input/serio/hyperv-keyboard.c @@ -75,8 +75,8 @@ struct synth_kbd_keystroke { #define HK_MAXIMUM_MESSAGE_SIZE 256 -#define KBD_VSC_SEND_RING_BUFFER_SIZE (40 * 1024) -#define KBD_VSC_RECV_RING_BUFFER_SIZE (40 * 1024) +#define KBD_VSC_SEND_RING_BUFFER_SIZE VMBUS_RING_SIZE(36 * 1024) +#define KBD_VSC_RECV_RING_BUFFER_SIZE VMBUS_RING_SIZE(36 * 1024) #define XTKBD_EMUL0 0xe0 #define XTKBD_EMUL1 0xe1 From d02a26991ecee772f8e510ce46ccac87ce480554 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Wed, 16 Sep 2020 11:48:15 +0800 Subject: [PATCH 191/265] HID: hyperv: Use VMBUS_RING_SIZE() for ringbuffer sizes For a Hyper-V vmbus, the size of the ringbuffer has two requirements: 1) it has to take one PAGE_SIZE for the header 2) it has to be PAGE_SIZE aligned so that double-mapping can work VMBUS_RING_SIZE() could calculate a correct ringbuffer size which fulfills both requirements, therefore use it to make sure vmbus work when PAGE_SIZE != HV_HYP_PAGE_SIZE (4K). Note that since the argument for VMBUS_RING_SIZE() is the size of payload (data part), so it will be minus 4k (the size of header when PAGE_SIZE = 4k) than the original value to keep the ringbuffer total size unchanged when PAGE_SIZE = 4k. Signed-off-by: Boqun Feng Cc: Jiri Kosina Cc: Michael Kelley Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200916034817.30282-10-boqun.feng@gmail.com Signed-off-by: Wei Liu --- drivers/hid/hid-hyperv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/hid/hid-hyperv.c b/drivers/hid/hid-hyperv.c index 0b6ee1dee625..978ee2aab2d4 100644 --- a/drivers/hid/hid-hyperv.c +++ b/drivers/hid/hid-hyperv.c @@ -104,8 +104,8 @@ struct synthhid_input_report { #pragma pack(pop) -#define INPUTVSC_SEND_RING_BUFFER_SIZE (40 * 1024) -#define INPUTVSC_RECV_RING_BUFFER_SIZE (40 * 1024) +#define INPUTVSC_SEND_RING_BUFFER_SIZE VMBUS_RING_SIZE(36 * 1024) +#define INPUTVSC_RECV_RING_BUFFER_SIZE VMBUS_RING_SIZE(36 * 1024) enum pipe_prot_msg_type { From 061dc93ef602dc8cdc719375538992b8f1e53478 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Wed, 16 Sep 2020 11:48:16 +0800 Subject: [PATCH 192/265] Driver: hv: util: Use VMBUS_RING_SIZE() for ringbuffer sizes For a Hyper-V vmbus, the size of the ringbuffer has two requirements: 1) it has to take one PAGE_SIZE for the header 2) it has to be PAGE_SIZE aligned so that double-mapping can work VMBUS_RING_SIZE() could calculate a correct ringbuffer size which fulfills both requirements, therefore use it to make sure vmbus work when PAGE_SIZE != HV_HYP_PAGE_SIZE (4K). Note that since the argument for VMBUS_RING_SIZE() is the size of payload (data part), so it will be minus 4k (the size of header when PAGE_SIZE = 4k) than the original value to keep the ringbuffer total size unchanged when PAGE_SIZE = 4k. Signed-off-by: Boqun Feng Cc: Michael Kelley Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200916034817.30282-11-boqun.feng@gmail.com Signed-off-by: Wei Liu --- drivers/hv/hv_util.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/hv/hv_util.c b/drivers/hv/hv_util.c index 92ee0fe4c919..f2845e919cc3 100644 --- a/drivers/hv/hv_util.c +++ b/drivers/hv/hv_util.c @@ -461,6 +461,9 @@ static void heartbeat_onchannelcallback(void *context) } } +#define HV_UTIL_RING_SEND_SIZE VMBUS_RING_SIZE(3 * HV_HYP_PAGE_SIZE) +#define HV_UTIL_RING_RECV_SIZE VMBUS_RING_SIZE(3 * HV_HYP_PAGE_SIZE) + static int util_probe(struct hv_device *dev, const struct hv_vmbus_device_id *dev_id) { @@ -491,8 +494,8 @@ static int util_probe(struct hv_device *dev, hv_set_drvdata(dev, srv); - ret = vmbus_open(dev->channel, 4 * HV_HYP_PAGE_SIZE, - 4 * HV_HYP_PAGE_SIZE, NULL, 0, srv->util_cb, + ret = vmbus_open(dev->channel, HV_UTIL_RING_SEND_SIZE, + HV_UTIL_RING_RECV_SIZE, NULL, 0, srv->util_cb, dev->channel); if (ret) goto error; @@ -551,8 +554,8 @@ static int util_resume(struct hv_device *dev) return ret; } - ret = vmbus_open(dev->channel, 4 * HV_HYP_PAGE_SIZE, - 4 * HV_HYP_PAGE_SIZE, NULL, 0, srv->util_cb, + ret = vmbus_open(dev->channel, HV_UTIL_RING_SEND_SIZE, + HV_UTIL_RING_RECV_SIZE, NULL, 0, srv->util_cb, dev->channel); return ret; } From 8f43710543ef305904c18072d14ace43c74b7743 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Wed, 16 Sep 2020 11:48:17 +0800 Subject: [PATCH 193/265] scsi: storvsc: Support PAGE_SIZE larger than 4K Hyper-V always use 4k page size (HV_HYP_PAGE_SIZE), so when communicating with Hyper-V, a guest should always use HV_HYP_PAGE_SIZE as the unit for page related data. For storvsc, the data is vmbus_packet_mpb_array. And since in scsi_cmnd, sglist of pages (in unit of PAGE_SIZE) is used, we need convert pages in the sglist of scsi_cmnd into Hyper-V pages in vmbus_packet_mpb_array. This patch does the conversion by dividing pages in sglist into Hyper-V pages, offset and indexes in vmbus_packet_mpb_array are recalculated accordingly. Signed-off-by: Boqun Feng Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200916034817.30282-12-boqun.feng@gmail.com Signed-off-by: Wei Liu --- drivers/scsi/storvsc_drv.c | 56 +++++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c index 8f5f5dc863a4..0c65fbd41035 100644 --- a/drivers/scsi/storvsc_drv.c +++ b/drivers/scsi/storvsc_drv.c @@ -1739,23 +1739,65 @@ static int storvsc_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scmnd) payload_sz = sizeof(cmd_request->mpb); if (sg_count) { - if (sg_count > MAX_PAGE_BUFFER_COUNT) { + unsigned int hvpgoff = 0; + unsigned long offset_in_hvpg = sgl->offset & ~HV_HYP_PAGE_MASK; + unsigned int hvpg_count = HVPFN_UP(offset_in_hvpg + length); + u64 hvpfn; - payload_sz = (sg_count * sizeof(u64) + + if (hvpg_count > MAX_PAGE_BUFFER_COUNT) { + + payload_sz = (hvpg_count * sizeof(u64) + sizeof(struct vmbus_packet_mpb_array)); payload = kzalloc(payload_sz, GFP_ATOMIC); if (!payload) return SCSI_MLQUEUE_DEVICE_BUSY; } + /* + * sgl is a list of PAGEs, and payload->range.pfn_array + * expects the page number in the unit of HV_HYP_PAGE_SIZE (the + * page size that Hyper-V uses, so here we need to divide PAGEs + * into HV_HYP_PAGE in case that PAGE_SIZE > HV_HYP_PAGE_SIZE. + * Besides, payload->range.offset should be the offset in one + * HV_HYP_PAGE. + */ payload->range.len = length; - payload->range.offset = sgl[0].offset; + payload->range.offset = offset_in_hvpg; + hvpgoff = sgl->offset >> HV_HYP_PAGE_SHIFT; cur_sgl = sgl; - for (i = 0; i < sg_count; i++) { - payload->range.pfn_array[i] = - page_to_pfn(sg_page((cur_sgl))); - cur_sgl = sg_next(cur_sgl); + for (i = 0; i < hvpg_count; i++) { + /* + * 'i' is the index of hv pages in the payload and + * 'hvpgoff' is the offset (in hv pages) of the first + * hv page in the the first page. The relationship + * between the sum of 'i' and 'hvpgoff' and the offset + * (in hv pages) in a payload page ('hvpgoff_in_page') + * is as follow: + * + * |------------------ PAGE -------------------| + * | NR_HV_HYP_PAGES_IN_PAGE hvpgs in total | + * |hvpg|hvpg| ... |hvpg|... |hvpg| + * ^ ^ ^ ^ + * +-hvpgoff-+ +-hvpgoff_in_page-+ + * ^ | + * +--------------------- i ---------------------------+ + */ + unsigned int hvpgoff_in_page = + (i + hvpgoff) % NR_HV_HYP_PAGES_IN_PAGE; + + /* + * Two cases that we need to fetch a page: + * 1) i == 0, the first step or + * 2) hvpgoff_in_page == 0, when we reach the boundary + * of a page. + */ + if (hvpgoff_in_page == 0 || i == 0) { + hvpfn = page_to_hvpfn(sg_page(cur_sgl)); + cur_sgl = sg_next(cur_sgl); + } + + payload->range.pfn_array[i] = hvpfn + hvpgoff_in_page; } } From 6d2730cb11214f492cbf381cef1758e15857bb69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Wilczy=C5=84ski?= Date: Fri, 25 Sep 2020 23:47:53 +0000 Subject: [PATCH 194/265] PCI: hv: Document missing hv_pci_protocol_negotiation() parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add missing documentation for the parameter "version" and "num_version" of the hv_pci_protocol_negotiation() function and resolve build time kernel-doc warnings: drivers/pci/controller/pci-hyperv.c:2535: warning: Function parameter or member 'version' not described in 'hv_pci_protocol_negotiation' drivers/pci/controller/pci-hyperv.c:2535: warning: Function parameter or member 'num_version' not described in 'hv_pci_protocol_negotiation' No change to functionality intended. Signed-off-by: Krzysztof WilczyÅ„ski Link: https://lore.kernel.org/r/20200925234753.1767227-1-kw@linux.com Reviewed-by: Michael Kelley Signed-off-by: Wei Liu --- drivers/pci/controller/pci-hyperv.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index fc4c3a15e570..6102330e27e1 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -2515,7 +2515,10 @@ static void hv_pci_onchannelcallback(void *context) /** * hv_pci_protocol_negotiation() - Set up protocol - * @hdev: VMBus's tracking struct for this root PCI bus + * @hdev: VMBus's tracking struct for this root PCI bus. + * @version: Array of supported channel protocol versions in + * the order of probing - highest go first. + * @num_version: Number of elements in the version array. * * This driver is intended to support running on Windows 10 * (server) and later versions. It will not run on earlier From e1471463180ddc36aa9970fe6a413feb0b608990 Mon Sep 17 00:00:00 2001 From: Joseph Salisbury Date: Sat, 26 Sep 2020 07:26:26 -0700 Subject: [PATCH 195/265] x86/hyperv: Remove aliases with X64 in their name In the architecture independent version of hyperv-tlfs.h, commit c55a844f46f958b removed the "X64" in the symbol names so they would make sense for both x86 and ARM64. That commit added aliases with the "X64" in the x86 version of hyperv-tlfs.h so that existing x86 code would continue to compile. As a cleanup, update the x86 code to use the symbols without the "X64", then remove the aliases. There's no functional change. Signed-off-by: Joseph Salisbury Link: https://lore.kernel.org/r/1601130386-11111-1-git-send-email-jsalisbury@linux.microsoft.com Reviewed-by: Michael Kelley Acked-by: Paolo Bonzini Signed-off-by: Wei Liu --- arch/x86/hyperv/hv_init.c | 8 ++++---- arch/x86/hyperv/hv_spinlock.c | 2 +- arch/x86/include/asm/hyperv-tlfs.h | 33 ------------------------------ arch/x86/kernel/cpu/mshyperv.c | 8 ++++---- arch/x86/kvm/hyperv.c | 20 +++++++++--------- 5 files changed, 19 insertions(+), 52 deletions(-) diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 6035df1b49e1..e04d90af4c27 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -148,9 +148,9 @@ static inline bool hv_reenlightenment_available(void) * Check for required features and priviliges to make TSC frequency * change notifications work. */ - return ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS && + return ms_hyperv.features & HV_ACCESS_FREQUENCY_MSRS && ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE && - ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT; + ms_hyperv.features & HV_ACCESS_REENLIGHTENMENT; } DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment) @@ -330,8 +330,8 @@ void __init hyperv_init(void) return; /* Absolutely required MSRs */ - required_msrs = HV_X64_MSR_HYPERCALL_AVAILABLE | - HV_X64_MSR_VP_INDEX_AVAILABLE; + required_msrs = HV_MSR_HYPERCALL_AVAILABLE | + HV_MSR_VP_INDEX_AVAILABLE; if ((ms_hyperv.features & required_msrs) != required_msrs) return; diff --git a/arch/x86/hyperv/hv_spinlock.c b/arch/x86/hyperv/hv_spinlock.c index 07f21a06392f..f3270c1fc48c 100644 --- a/arch/x86/hyperv/hv_spinlock.c +++ b/arch/x86/hyperv/hv_spinlock.c @@ -66,7 +66,7 @@ void __init hv_init_spinlocks(void) { if (!hv_pvspin || !apic || !(ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) || - !(ms_hyperv.features & HV_X64_MSR_GUEST_IDLE_AVAILABLE)) { + !(ms_hyperv.features & HV_MSR_GUEST_IDLE_AVAILABLE)) { pr_info("PV spinlocks disabled\n"); return; } diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h index 7a4d2062385c..0ed20e8bba9e 100644 --- a/arch/x86/include/asm/hyperv-tlfs.h +++ b/arch/x86/include/asm/hyperv-tlfs.h @@ -27,39 +27,6 @@ #define HYPERV_CPUID_MIN 0x40000005 #define HYPERV_CPUID_MAX 0x4000ffff -/* - * Aliases for Group A features that have X64 in the name. - * On x86/x64 these are HYPERV_CPUID_FEATURES.EAX bits. - */ - -#define HV_X64_MSR_VP_RUNTIME_AVAILABLE \ - HV_MSR_VP_RUNTIME_AVAILABLE -#define HV_X64_MSR_SYNIC_AVAILABLE \ - HV_MSR_SYNIC_AVAILABLE -#define HV_X64_MSR_APIC_ACCESS_AVAILABLE \ - HV_MSR_APIC_ACCESS_AVAILABLE -#define HV_X64_MSR_HYPERCALL_AVAILABLE \ - HV_MSR_HYPERCALL_AVAILABLE -#define HV_X64_MSR_VP_INDEX_AVAILABLE \ - HV_MSR_VP_INDEX_AVAILABLE -#define HV_X64_MSR_RESET_AVAILABLE \ - HV_MSR_RESET_AVAILABLE -#define HV_X64_MSR_GUEST_IDLE_AVAILABLE \ - HV_MSR_GUEST_IDLE_AVAILABLE -#define HV_X64_ACCESS_FREQUENCY_MSRS \ - HV_ACCESS_FREQUENCY_MSRS -#define HV_X64_ACCESS_REENLIGHTENMENT \ - HV_ACCESS_REENLIGHTENMENT -#define HV_X64_ACCESS_TSC_INVARIANT \ - HV_ACCESS_TSC_INVARIANT - -/* - * Aliases for Group B features that have X64 in the name. - * On x86/x64 these are HYPERV_CPUID_FEATURES.EBX bits. - */ -#define HV_X64_POST_MESSAGES HV_POST_MESSAGES -#define HV_X64_SIGNAL_EVENTS HV_SIGNAL_EVENTS - /* * Group D Features. The bit assignments are custom to each architecture. * On x86/x64 these are HYPERV_CPUID_FEATURES.EDX bits. diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index 31125448b174..9834a43cd0fa 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -248,7 +248,7 @@ static void __init ms_hyperv_init_platform(void) hv_host_info_edx >> 24, hv_host_info_edx & 0xFFFFFF); } - if (ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS && + if (ms_hyperv.features & HV_ACCESS_FREQUENCY_MSRS && ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE) { x86_platform.calibrate_tsc = hv_get_tsc_khz; x86_platform.calibrate_cpu = hv_get_tsc_khz; @@ -270,7 +270,7 @@ static void __init ms_hyperv_init_platform(void) crash_kexec_post_notifiers = true; #ifdef CONFIG_X86_LOCAL_APIC - if (ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS && + if (ms_hyperv.features & HV_ACCESS_FREQUENCY_MSRS && ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE) { /* * Get the APIC frequency. @@ -296,7 +296,7 @@ static void __init ms_hyperv_init_platform(void) machine_ops.shutdown = hv_machine_shutdown; machine_ops.crash_shutdown = hv_machine_crash_shutdown; #endif - if (ms_hyperv.features & HV_X64_ACCESS_TSC_INVARIANT) { + if (ms_hyperv.features & HV_ACCESS_TSC_INVARIANT) { wrmsrl(HV_X64_MSR_TSC_INVARIANT_CONTROL, 0x1); setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE); } else { @@ -330,7 +330,7 @@ static void __init ms_hyperv_init_platform(void) alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_hyperv_callback); /* Setup the IDT for reenlightenment notifications */ - if (ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT) { + if (ms_hyperv.features & HV_ACCESS_REENLIGHTENMENT) { alloc_intr_gate(HYPERV_REENLIGHTENMENT_VECTOR, asm_sysvec_hyperv_reenlightenment); } diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 814d3aee5cef..21d375a9c4da 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -2000,20 +2000,20 @@ int kvm_vcpu_ioctl_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, break; case HYPERV_CPUID_FEATURES: - ent->eax |= HV_X64_MSR_VP_RUNTIME_AVAILABLE; + ent->eax |= HV_MSR_VP_RUNTIME_AVAILABLE; ent->eax |= HV_MSR_TIME_REF_COUNT_AVAILABLE; - ent->eax |= HV_X64_MSR_SYNIC_AVAILABLE; + ent->eax |= HV_MSR_SYNIC_AVAILABLE; ent->eax |= HV_MSR_SYNTIMER_AVAILABLE; - ent->eax |= HV_X64_MSR_APIC_ACCESS_AVAILABLE; - ent->eax |= HV_X64_MSR_HYPERCALL_AVAILABLE; - ent->eax |= HV_X64_MSR_VP_INDEX_AVAILABLE; - ent->eax |= HV_X64_MSR_RESET_AVAILABLE; + ent->eax |= HV_MSR_APIC_ACCESS_AVAILABLE; + ent->eax |= HV_MSR_HYPERCALL_AVAILABLE; + ent->eax |= HV_MSR_VP_INDEX_AVAILABLE; + ent->eax |= HV_MSR_RESET_AVAILABLE; ent->eax |= HV_MSR_REFERENCE_TSC_AVAILABLE; - ent->eax |= HV_X64_ACCESS_FREQUENCY_MSRS; - ent->eax |= HV_X64_ACCESS_REENLIGHTENMENT; + ent->eax |= HV_ACCESS_FREQUENCY_MSRS; + ent->eax |= HV_ACCESS_REENLIGHTENMENT; - ent->ebx |= HV_X64_POST_MESSAGES; - ent->ebx |= HV_X64_SIGNAL_EVENTS; + ent->ebx |= HV_POST_MESSAGES; + ent->ebx |= HV_SIGNAL_EVENTS; ent->edx |= HV_FEATURE_FREQUENCY_MSRS_AVAILABLE; ent->edx |= HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; From 1f3aed01473c41c9f896fbf4c30d330655e8aa7c Mon Sep 17 00:00:00 2001 From: Mohammed Gamal Date: Thu, 24 Sep 2020 17:11:17 +0200 Subject: [PATCH 196/265] hv: clocksource: Add notrace attribute to read_hv_sched_clock_*() functions When selecting function_graph tracer with the command: # echo function_graph > /sys/kernel/debug/tracing/current_tracer The kernel crashes with the following stack trace: [69703.122389] BUG: stack guard page was hit at 000000001056545c (stack is 00000000fa3f8fed..0000000005d39503) [69703.122403] kernel stack overflow (double-fault): 0000 [#1] SMP PTI [69703.122413] CPU: 0 PID: 16982 Comm: bash Kdump: loaded Not tainted 4.18.0-236.el8.x86_64 #1 [69703.122420] Hardware name: Microsoft Corporation Virtual Machine/Virtual Machine, BIOS Hyper-V UEFI Release v4.0 12/17/2019 [69703.122433] RIP: 0010repare_ftrace_return+0xa/0x110 [69703.122458] Code: 05 00 0f 0b 48 c7 c7 10 ca 69 ae 0f b6 f0 e8 4b 52 0c 00 31 c0 eb ca 66 0f 1f 84 00 00 00 00 00 55 48 89 e5 41 56 41 55 41 54 <53> 48 83 ec 18 65 48 8b 04 25 28 00 00 00 48 89 45 d8 31 c0 48 85 [69703.122467] RSP: 0018:ffffbd6d01118000 EFLAGS: 00010086 [69703.122476] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000003 [69703.122484] RDX: 0000000000000000 RSI: ffffbd6d011180d8 RDI: ffffffffadce7550 [69703.122491] RBP: ffffbd6d01118018 R08: 0000000000000000 R09: ffff9d4b09266000 [69703.122498] R10: ffff9d4b0fc04540 R11: ffff9d4b0fc20a00 R12: ffff9d4b6e42aa90 [69703.122506] R13: ffff9d4b0fc20ab8 R14: 00000000000003e8 R15: ffffbd6d0111837c [69703.122514] FS: 00007fd5f2588740(0000) GS:ffff9d4b6e400000(0000) knlGS:0000000000000000 [69703.122521] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [69703.122528] CR2: ffffbd6d01117ff8 CR3: 00000000565d8001 CR4: 00000000003606f0 [69703.122538] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [69703.122545] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [69703.122552] Call Trace: [69703.122568] ftrace_graph_caller+0x6b/0xa0 [69703.122589] ? read_hv_sched_clock_tsc+0x5/0x20 [69703.122599] read_hv_sched_clock_tsc+0x5/0x20 [69703.122611] sched_clock+0x5/0x10 [69703.122621] sched_clock_local+0x12/0x80 [69703.122631] sched_clock_cpu+0x8c/0xb0 [69703.122644] trace_clock_global+0x21/0x90 [69703.122655] ring_buffer_lock_reserve+0x100/0x3c0 [69703.122671] trace_buffer_lock_reserve+0x16/0x50 [69703.122683] __trace_graph_entry+0x28/0x90 [69703.122695] trace_graph_entry+0xfd/0x1a0 [69703.122705] ? read_hv_clock_tsc_cs+0x10/0x10 [69703.122714] ? sched_clock+0x5/0x10 [69703.122723] prepare_ftrace_return+0x99/0x110 [69703.122734] ? read_hv_clock_tsc_cs+0x10/0x10 [69703.122743] ? sched_clock+0x5/0x10 [69703.122752] ftrace_graph_caller+0x6b/0xa0 [69703.122768] ? read_hv_clock_tsc_cs+0x10/0x10 [69703.122777] ? sched_clock+0x5/0x10 [69703.122786] ? read_hv_sched_clock_tsc+0x5/0x20 [69703.122796] ? ring_buffer_unlock_commit+0x1d/0xa0 [69703.122805] read_hv_sched_clock_tsc+0x5/0x20 [69703.122814] ftrace_graph_caller+0xa0/0xa0 [ ... recursion snipped ... ] Setting the notrace attribute for read_hv_sched_clock_msr() and read_hv_sched_clock_tsc() fixes it. Fixes: bd00cd52d5be ("clocksource/drivers/hyperv: Add Hyper-V specific sched clock function") Signed-off-by: Vitaly Kuznetsov Signed-off-by: Mohammed Gamal Link: https://lore.kernel.org/r/20200924151117.767442-1-mgamal@redhat.com Signed-off-by: Wei Liu --- drivers/clocksource/hyperv_timer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/clocksource/hyperv_timer.c b/drivers/clocksource/hyperv_timer.c index 09aa44cb8a91..ba04cb381cd3 100644 --- a/drivers/clocksource/hyperv_timer.c +++ b/drivers/clocksource/hyperv_timer.c @@ -341,7 +341,7 @@ static u64 notrace read_hv_clock_tsc_cs(struct clocksource *arg) return read_hv_clock_tsc(); } -static u64 read_hv_sched_clock_tsc(void) +static u64 notrace read_hv_sched_clock_tsc(void) { return (read_hv_clock_tsc() - hv_sched_clock_offset) * (NSEC_PER_SEC / HV_CLOCK_HZ); @@ -404,7 +404,7 @@ static u64 notrace read_hv_clock_msr_cs(struct clocksource *arg) return read_hv_clock_msr(); } -static u64 read_hv_sched_clock_msr(void) +static u64 notrace read_hv_sched_clock_msr(void) { return (read_hv_clock_msr() - hv_sched_clock_offset) * (NSEC_PER_SEC / HV_CLOCK_HZ); From 55c4478a8f0ecedc0c1a0c9379380249985c372a Mon Sep 17 00:00:00 2001 From: Xiaoyi Chen Date: Tue, 22 Sep 2020 16:19:19 +0000 Subject: [PATCH 197/265] PM: hibernate: Batch hibernate and resume IO requests Hibernate and resume process submits individual IO requests for each page of the data, so use blk_plug to improve the batching of these requests. Testing this change with hibernate and resumes consistently shows merging of the IO requests and more than an order of magnitude improvement in hibernate and resume speed is observed. One hibernate and resume cycle for 16GB RAM out of 32GB in use takes around 21 minutes before the change, and 1 minutes after the change on a system with limited storage IOPS. Signed-off-by: Xiaoyi Chen Co-Developed-by: Anchal Agarwal Signed-off-by: Anchal Agarwal [ rjw: Subject and changelog edits, white space damage fixes ] Signed-off-by: Rafael J. Wysocki --- kernel/power/swap.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 01e2858b5fe3..116320a0394d 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -226,6 +226,7 @@ struct hib_bio_batch { atomic_t count; wait_queue_head_t wait; blk_status_t error; + struct blk_plug plug; }; static void hib_init_batch(struct hib_bio_batch *hb) @@ -233,6 +234,12 @@ static void hib_init_batch(struct hib_bio_batch *hb) atomic_set(&hb->count, 0); init_waitqueue_head(&hb->wait); hb->error = BLK_STS_OK; + blk_start_plug(&hb->plug); +} + +static void hib_finish_batch(struct hib_bio_batch *hb) +{ + blk_finish_plug(&hb->plug); } static void hib_end_io(struct bio *bio) @@ -294,6 +301,10 @@ static int hib_submit_io(int op, int op_flags, pgoff_t page_off, void *addr, static blk_status_t hib_wait_io(struct hib_bio_batch *hb) { + /* + * We are relying on the behavior of blk_plug that a thread with + * a plug will flush the plug list before sleeping. + */ wait_event(hb->wait, atomic_read(&hb->count) == 0); return blk_status_to_errno(hb->error); } @@ -561,6 +572,7 @@ static int save_image(struct swap_map_handle *handle, nr_pages++; } err2 = hib_wait_io(&hb); + hib_finish_batch(&hb); stop = ktime_get(); if (!ret) ret = err2; @@ -854,6 +866,7 @@ out_finish: pr_info("Image saving done\n"); swsusp_show_speed(start, stop, nr_to_write, "Wrote"); out_clean: + hib_finish_batch(&hb); if (crc) { if (crc->thr) kthread_stop(crc->thr); @@ -1084,6 +1097,7 @@ static int load_image(struct swap_map_handle *handle, nr_pages++; } err2 = hib_wait_io(&hb); + hib_finish_batch(&hb); stop = ktime_get(); if (!ret) ret = err2; @@ -1447,6 +1461,7 @@ out_finish: } swsusp_show_speed(start, stop, nr_to_read, "Read"); out_clean: + hib_finish_batch(&hb); for (i = 0; i < ring_size; i++) free_page((unsigned long)page[i]); if (crc) { From 6b61d49a55796dbbc479eeb4465e59fd656c719c Mon Sep 17 00:00:00 2001 From: Grygorii Strashko Date: Fri, 18 Sep 2020 19:55:18 +0300 Subject: [PATCH 198/265] PM: runtime: Fix timer_expires data type on 32-bit arches Commit 8234f6734c5d ("PM-runtime: Switch autosuspend over to using hrtimers") switched PM runtime autosuspend to use hrtimers and all related time accounting in ns, but missed to update the timer_expires data type in struct dev_pm_info to u64. This causes the timer_expires value to be truncated on 32-bit architectures when assignment is done from u64 values: rpm_suspend() |- dev->power.timer_expires = expires; Fix it by changing the timer_expires type to u64. Fixes: 8234f6734c5d ("PM-runtime: Switch autosuspend over to using hrtimers") Signed-off-by: Grygorii Strashko Acked-by: Pavel Machek Acked-by: Vincent Guittot Cc: 5.0+ # 5.0+ [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- include/linux/pm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/pm.h b/include/linux/pm.h index a30a4b54df52..47aca6bac1d6 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -590,7 +590,7 @@ struct dev_pm_info { #endif #ifdef CONFIG_PM struct hrtimer suspend_timer; - unsigned long timer_expires; + u64 timer_expires; struct work_struct work; wait_queue_head_t wait_queue; struct wake_irq *wakeirq; From 7b38b7b0427df70237e3c9c73f2db6b99be2b4b9 Mon Sep 17 00:00:00 2001 From: Leonard Crestez Date: Tue, 8 Sep 2020 19:24:45 +0900 Subject: [PATCH 199/265] PM / devfreq: Add devfreq_get_devfreq_by_node function Split off part of devfreq_get_devfreq_by_phandle into a separate function. This allows callers to fetch devfreq instances by enumerating devicetree instead of explicit phandles. Acked-by: Krzysztof Kozlowski Reviewed-by: Lukasz Luba Signed-off-by: Leonard Crestez [cw00.choi: Export devfreq_get_devfreq_by_node function and add function to devfreq.h when CONFIG_PM_DEVFREQ is enabled.] Signed-off-by: Chanwoo Choi --- drivers/devfreq/devfreq.c | 46 +++++++++++++++++++++++++++++---------- include/linux/devfreq.h | 6 +++++ 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index 071b59fe84d2..d4424b5d8306 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -983,6 +983,32 @@ struct devfreq *devm_devfreq_add_device(struct device *dev, EXPORT_SYMBOL(devm_devfreq_add_device); #ifdef CONFIG_OF +/* + * devfreq_get_devfreq_by_node - Get the devfreq device from devicetree + * @node - pointer to device_node + * + * return the instance of devfreq device + */ +struct devfreq *devfreq_get_devfreq_by_node(struct device_node *node) +{ + struct devfreq *devfreq; + + if (!node) + return ERR_PTR(-EINVAL); + + mutex_lock(&devfreq_list_lock); + list_for_each_entry(devfreq, &devfreq_list, node) { + if (devfreq->dev.parent + && devfreq->dev.parent->of_node == node) { + mutex_unlock(&devfreq_list_lock); + return devfreq; + } + } + mutex_unlock(&devfreq_list_lock); + + return ERR_PTR(-ENODEV); +} + /* * devfreq_get_devfreq_by_phandle - Get the devfreq device from devicetree * @dev - instance to the given device @@ -1005,26 +1031,24 @@ struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, int index) if (!node) return ERR_PTR(-ENODEV); - mutex_lock(&devfreq_list_lock); - list_for_each_entry(devfreq, &devfreq_list, node) { - if (devfreq->dev.parent - && devfreq->dev.parent->of_node == node) { - mutex_unlock(&devfreq_list_lock); - of_node_put(node); - return devfreq; - } - } - mutex_unlock(&devfreq_list_lock); + devfreq = devfreq_get_devfreq_by_node(node); of_node_put(node); - return ERR_PTR(-EPROBE_DEFER); + return devfreq; } + #else +struct devfreq *devfreq_get_devfreq_by_node(struct device_node *node) +{ + return ERR_PTR(-ENODEV); +} + struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, int index) { return ERR_PTR(-ENODEV); } #endif /* CONFIG_OF */ +EXPORT_SYMBOL_GPL(devfreq_get_devfreq_by_node); EXPORT_SYMBOL_GPL(devfreq_get_devfreq_by_phandle); /** diff --git a/include/linux/devfreq.h b/include/linux/devfreq.h index 12782fbb4c25..eb971b8e5051 100644 --- a/include/linux/devfreq.h +++ b/include/linux/devfreq.h @@ -261,6 +261,7 @@ void devm_devfreq_unregister_notifier(struct device *dev, struct devfreq *devfreq, struct notifier_block *nb, unsigned int list); +struct devfreq *devfreq_get_devfreq_by_node(struct device_node *node); struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, int index); #if IS_ENABLED(CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND) @@ -414,6 +415,11 @@ static inline void devm_devfreq_unregister_notifier(struct device *dev, { } +static inline struct devfreq *devfreq_get_devfreq_by_node(struct device_node *node) +{ + return ERR_PTR(-ENODEV); +} + static inline struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, int index) { From 86d90fd95bbc3b3fdc2ef0507b7324cd1d0a358b Mon Sep 17 00:00:00 2001 From: Chanwoo Choi Date: Tue, 8 Sep 2020 19:24:46 +0900 Subject: [PATCH 200/265] PM / devfreq: Change prototype of devfreq_get_devfreq_by_phandle function Previously, devfreq core support 'devfreq' property in order to get the devfreq device by phandle. But, 'devfreq' property name is not proper on devicetree binding because this name doesn't mean the any h/w attribute. The devfreq core hand over the right to decide the property name for getting the devfreq device on devicetree. Each devfreq driver will decide the property name on devicetree binding and pass the their own property name to devfreq_get_devfreq_by_phandle function. Acked-by: Krzysztof Kozlowski Signed-off-by: Chanwoo Choi --- drivers/devfreq/devfreq.c | 11 +++++++---- drivers/devfreq/exynos-bus.c | 2 +- include/linux/devfreq.h | 5 +++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index d4424b5d8306..861c100f9fac 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -1012,22 +1012,24 @@ struct devfreq *devfreq_get_devfreq_by_node(struct device_node *node) /* * devfreq_get_devfreq_by_phandle - Get the devfreq device from devicetree * @dev - instance to the given device + * @phandle_name - name of property holding a phandle value * @index - index into list of devfreq * * return the instance of devfreq device */ -struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, int index) +struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, + const char *phandle_name, int index) { struct device_node *node; struct devfreq *devfreq; - if (!dev) + if (!dev || !phandle_name) return ERR_PTR(-EINVAL); if (!dev->of_node) return ERR_PTR(-EINVAL); - node = of_parse_phandle(dev->of_node, "devfreq", index); + node = of_parse_phandle(dev->of_node, phandle_name, index); if (!node) return ERR_PTR(-ENODEV); @@ -1043,7 +1045,8 @@ struct devfreq *devfreq_get_devfreq_by_node(struct device_node *node) return ERR_PTR(-ENODEV); } -struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, int index) +struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, + const char *phandle_name, int index) { return ERR_PTR(-ENODEV); } diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c index 8fa8eb541373..58dbf51f0983 100644 --- a/drivers/devfreq/exynos-bus.c +++ b/drivers/devfreq/exynos-bus.c @@ -360,7 +360,7 @@ static int exynos_bus_profile_init_passive(struct exynos_bus *bus, profile->exit = exynos_bus_passive_exit; /* Get the instance of parent devfreq device */ - parent_devfreq = devfreq_get_devfreq_by_phandle(dev, 0); + parent_devfreq = devfreq_get_devfreq_by_phandle(dev, "devfreq", 0); if (IS_ERR(parent_devfreq)) return -EPROBE_DEFER; diff --git a/include/linux/devfreq.h b/include/linux/devfreq.h index eb971b8e5051..2f4a74efa6be 100644 --- a/include/linux/devfreq.h +++ b/include/linux/devfreq.h @@ -262,7 +262,8 @@ void devm_devfreq_unregister_notifier(struct device *dev, struct notifier_block *nb, unsigned int list); struct devfreq *devfreq_get_devfreq_by_node(struct device_node *node); -struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, int index); +struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, + const char *phandle_name, int index); #if IS_ENABLED(CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND) /** @@ -421,7 +422,7 @@ static inline struct devfreq *devfreq_get_devfreq_by_node(struct device_node *no } static inline struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, - int index) + const char *phandle_name, int index) { return ERR_PTR(-ENODEV); } From 02bdbf7d09c059b16047e3d4a05e6d584dd993b6 Mon Sep 17 00:00:00 2001 From: Chanwoo Choi Date: Tue, 8 Sep 2020 19:24:47 +0900 Subject: [PATCH 201/265] PM / devfreq: event: Change prototype of devfreq_event_get_edev_by_phandle function Previously, devfreq core support 'devfreq-events' property in order to get the devfreq-event device by phandle. But, 'devfreq-events' property name is not proper on devicetree binding because this name doesn't mean the any h/w attribute. The devfreq-event core hand over the rights to decide the property name for getting the devfreq-event device on devicetree. Each devfreq-event driver will decide the property name on devicetree binding and then pass the their own property name to devfreq_event_get_edev_by_phandle function. And change the prototype of devfreq_event_get_edev_count function because of used deprecated 'devfreq-events' property. Acked-by: Krzysztof Kozlowski Signed-off-by: Chanwoo Choi --- drivers/devfreq/devfreq-event.c | 14 ++++++++------ drivers/devfreq/exynos-bus.c | 5 +++-- drivers/devfreq/rk3399_dmc.c | 2 +- drivers/memory/samsung/exynos5422-dmc.c | 6 ++++-- include/linux/devfreq-event.h | 14 ++++++++++---- 5 files changed, 26 insertions(+), 15 deletions(-) diff --git a/drivers/devfreq/devfreq-event.c b/drivers/devfreq/devfreq-event.c index 56efbeb7851e..6765c03334bc 100644 --- a/drivers/devfreq/devfreq-event.c +++ b/drivers/devfreq/devfreq-event.c @@ -213,20 +213,21 @@ EXPORT_SYMBOL_GPL(devfreq_event_reset_event); * devfreq_event_get_edev_by_phandle() - Get the devfreq-event dev from * devicetree. * @dev : the pointer to the given device + * @phandle_name: name of property holding a phandle value * @index : the index into list of devfreq-event device * * Note that this function return the pointer of devfreq-event device. */ struct devfreq_event_dev *devfreq_event_get_edev_by_phandle(struct device *dev, - int index) + const char *phandle_name, int index) { struct device_node *node; struct devfreq_event_dev *edev; - if (!dev->of_node) + if (!dev->of_node || !phandle_name) return ERR_PTR(-EINVAL); - node = of_parse_phandle(dev->of_node, "devfreq-events", index); + node = of_parse_phandle(dev->of_node, phandle_name, index); if (!node) return ERR_PTR(-ENODEV); @@ -258,19 +259,20 @@ EXPORT_SYMBOL_GPL(devfreq_event_get_edev_by_phandle); /** * devfreq_event_get_edev_count() - Get the count of devfreq-event dev * @dev : the pointer to the given device + * @phandle_name: name of property holding a phandle value * * Note that this function return the count of devfreq-event devices. */ -int devfreq_event_get_edev_count(struct device *dev) +int devfreq_event_get_edev_count(struct device *dev, const char *phandle_name) { int count; - if (!dev->of_node) { + if (!dev->of_node || !phandle_name) { dev_err(dev, "device does not have a device node entry\n"); return -EINVAL; } - count = of_property_count_elems_of_size(dev->of_node, "devfreq-events", + count = of_property_count_elems_of_size(dev->of_node, phandle_name, sizeof(u32)); if (count < 0) { dev_err(dev, diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c index 58dbf51f0983..1e684a448c9e 100644 --- a/drivers/devfreq/exynos-bus.c +++ b/drivers/devfreq/exynos-bus.c @@ -193,7 +193,7 @@ static int exynos_bus_parent_parse_of(struct device_node *np, * Get the devfreq-event devices to get the current utilization of * buses. This raw data will be used in devfreq ondemand governor. */ - count = devfreq_event_get_edev_count(dev); + count = devfreq_event_get_edev_count(dev, "devfreq-events"); if (count < 0) { dev_err(dev, "failed to get the count of devfreq-event dev\n"); ret = count; @@ -209,7 +209,8 @@ static int exynos_bus_parent_parse_of(struct device_node *np, } for (i = 0; i < count; i++) { - bus->edev[i] = devfreq_event_get_edev_by_phandle(dev, i); + bus->edev[i] = devfreq_event_get_edev_by_phandle(dev, + "devfreq-events", i); if (IS_ERR(bus->edev[i])) { ret = -EPROBE_DEFER; goto err_regulator; diff --git a/drivers/devfreq/rk3399_dmc.c b/drivers/devfreq/rk3399_dmc.c index 027769e39f9b..2e912166a993 100644 --- a/drivers/devfreq/rk3399_dmc.c +++ b/drivers/devfreq/rk3399_dmc.c @@ -341,7 +341,7 @@ static int rk3399_dmcfreq_probe(struct platform_device *pdev) return PTR_ERR(data->dmc_clk); } - data->edev = devfreq_event_get_edev_by_phandle(dev, 0); + data->edev = devfreq_event_get_edev_by_phandle(dev, "devfreq-events", 0); if (IS_ERR(data->edev)) return -EPROBE_DEFER; diff --git a/drivers/memory/samsung/exynos5422-dmc.c b/drivers/memory/samsung/exynos5422-dmc.c index b9c7956e5031..714d1f6f077c 100644 --- a/drivers/memory/samsung/exynos5422-dmc.c +++ b/drivers/memory/samsung/exynos5422-dmc.c @@ -1293,7 +1293,8 @@ static int exynos5_performance_counters_init(struct exynos5_dmc *dmc) int counters_size; int ret, i; - dmc->num_counters = devfreq_event_get_edev_count(dmc->dev); + dmc->num_counters = devfreq_event_get_edev_count(dmc->dev, + "devfreq-events"); if (dmc->num_counters < 0) { dev_err(dmc->dev, "could not get devfreq-event counters\n"); return dmc->num_counters; @@ -1306,7 +1307,8 @@ static int exynos5_performance_counters_init(struct exynos5_dmc *dmc) for (i = 0; i < dmc->num_counters; i++) { dmc->counter[i] = - devfreq_event_get_edev_by_phandle(dmc->dev, i); + devfreq_event_get_edev_by_phandle(dmc->dev, + "devfreq-events", i); if (IS_ERR_OR_NULL(dmc->counter[i])) return -EPROBE_DEFER; } diff --git a/include/linux/devfreq-event.h b/include/linux/devfreq-event.h index f14f17f8cb7f..4a50a5c71a5f 100644 --- a/include/linux/devfreq-event.h +++ b/include/linux/devfreq-event.h @@ -106,8 +106,11 @@ extern int devfreq_event_get_event(struct devfreq_event_dev *edev, struct devfreq_event_data *edata); extern int devfreq_event_reset_event(struct devfreq_event_dev *edev); extern struct devfreq_event_dev *devfreq_event_get_edev_by_phandle( - struct device *dev, int index); -extern int devfreq_event_get_edev_count(struct device *dev); + struct device *dev, + const char *phandle_name, + int index); +extern int devfreq_event_get_edev_count(struct device *dev, + const char *phandle_name); extern struct devfreq_event_dev *devfreq_event_add_edev(struct device *dev, struct devfreq_event_desc *desc); extern int devfreq_event_remove_edev(struct devfreq_event_dev *edev); @@ -152,12 +155,15 @@ static inline int devfreq_event_reset_event(struct devfreq_event_dev *edev) } static inline struct devfreq_event_dev *devfreq_event_get_edev_by_phandle( - struct device *dev, int index) + struct device *dev, + const char *phandle_name, + int index) { return ERR_PTR(-EINVAL); } -static inline int devfreq_event_get_edev_count(struct device *dev) +static inline int devfreq_event_get_edev_count(struct device *dev, + const char *phandle_name) { return -EINVAL; } From d353d1202b89ab039acd079cd97f7646058ebe11 Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Sun, 27 Sep 2020 23:51:39 +0300 Subject: [PATCH 202/265] PM / devfreq: tegra30: Improve initial hardware resetting It's safe to enable the ACTMON clock at any time during driver probing, even if we don't know the state of hardware, because it's used only for collecting and processing stats, and interrupt is kept disabled. This allows us to slightly improve code which performs initial hardware resetting by making use of a single reset_control_reset() instead of assert/deassert pair. Secondly, a potential error of the reset-control API is handled nicely now. Signed-off-by: Dmitry Osipenko Signed-off-by: Chanwoo Choi --- drivers/devfreq/tegra30-devfreq.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/devfreq/tegra30-devfreq.c b/drivers/devfreq/tegra30-devfreq.c index dedd39de7367..f5e74c2ede85 100644 --- a/drivers/devfreq/tegra30-devfreq.c +++ b/drivers/devfreq/tegra30-devfreq.c @@ -822,8 +822,6 @@ static int tegra_devfreq_probe(struct platform_device *pdev) return err; } - reset_control_assert(tegra->reset); - err = clk_prepare_enable(tegra->clock); if (err) { dev_err(&pdev->dev, @@ -831,7 +829,11 @@ static int tegra_devfreq_probe(struct platform_device *pdev) return err; } - reset_control_deassert(tegra->reset); + err = reset_control_reset(tegra->reset); + if (err) { + dev_err(&pdev->dev, "Failed to reset hardware: %d\n", err); + goto disable_clk; + } rate = clk_round_rate(tegra->emc_clock, ULONG_MAX); if (rate < 0) { From 4849bc777049b568a35d5d63ae326e93f0fff9de Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 28 Sep 2020 12:45:55 -0700 Subject: [PATCH 203/265] ACPI / NUMA: Add stub function for pxm_to_node() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After commit 01feba590cd6 ("ACPI: Do not create new NUMA domains from ACPI static tables that are not SRAT"): $ scripts/config --file arch/x86/configs/x86_64_defconfig -d NUMA -e ACPI_NFIT $ make -skj"$(nproc)" distclean defconfig drivers/acpi/nfit/ drivers/acpi/nfit/core.c: In function ‘acpi_nfit_register_region’: drivers/acpi/nfit/core.c:3010:27: error: implicit declaration of function ‘pxm_to_node’; did you mean ‘xa_to_node’? [-Werror=implicit-function-declaration] 3010 | ndr_desc->target_node = pxm_to_node(spa->proximity_domain); | ^~~~~~~~~~~ | xa_to_node cc1: some warnings being treated as errors ... Add a stub function like acpi_map_pxm_to_node() had so that the build continues to work. Fixes: 01feba590cd6 ("ACPI: Do not create new NUMA domains from ACPI static tables that are not SRAT") Signed-off-by: Nathan Chancellor Acked-by: Randy Dunlap # build-tested Acked-by: Jonathan Cameron Reviewed-by: Hanjun Guo Signed-off-by: Rafael J. Wysocki --- include/acpi/acpi_numa.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/acpi/acpi_numa.h b/include/acpi/acpi_numa.h index fdebcfc6c8df..09eb3bc20ff5 100644 --- a/include/acpi/acpi_numa.h +++ b/include/acpi/acpi_numa.h @@ -22,5 +22,10 @@ extern int acpi_numa __initdata; extern void bad_srat(void); extern int srat_disabled(void); +#else /* CONFIG_ACPI_NUMA */ +static inline int pxm_to_node(int pxm) +{ + return 0; +} #endif /* CONFIG_ACPI_NUMA */ #endif /* __ACP_NUMA_H */ From b0b8e56b82c06b3bb6e5fb66d0e9c9c3fd3ce555 Mon Sep 17 00:00:00 2001 From: Jann Horn Date: Tue, 29 Sep 2020 00:49:16 +0200 Subject: [PATCH 204/265] objtool: Permit __kasan_check_{read,write} under UACCESS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Building linux-next with JUMP_LABEL=n and KASAN=y, I got this objtool warning: arch/x86/lib/copy_mc.o: warning: objtool: copy_mc_to_user()+0x22: call to __kasan_check_read() with UACCESS enabled What happens here is that copy_mc_to_user() branches on a static key in a UACCESS region:         __uaccess_begin();         if (static_branch_unlikely(©_mc_fragile_key))                 ret = copy_mc_fragile(to, from, len);         ret = copy_mc_generic(to, from, len);         __uaccess_end(); and the !CONFIG_JUMP_LABEL version of static_branch_unlikely() uses static_key_enabled(), which uses static_key_count(), which uses atomic_read(), which calls instrument_atomic_read(), which uses kasan_check_read(), which is __kasan_check_read(). Let's permit these KASAN helpers in UACCESS regions - static keys should probably work under UACCESS, I think. PeterZ adds: It's not a matter of permitting, it's a matter of being safe and correct. In this case it is, because it's a thin wrapper around check_memory_region() which was already marked safe. check_memory_region() is correct because the only thing it ends up calling is kasa_report() and that is also marked safe because that is annotated with user_access_save/restore() before it does anything else. On top of that, all of KASAN is noinstr, so nothing in here will end up in tracing and/or call schedule() before the user_access_save(). Signed-off-by: Jann Horn Acked-by: Peter Zijlstra (Intel) Signed-off-by: Josh Poimboeuf --- tools/objtool/check.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 2df9f769412e..3d14134c4e97 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -583,6 +583,8 @@ static const char *uaccess_safe_builtin[] = { "__asan_store4_noabort", "__asan_store8_noabort", "__asan_store16_noabort", + "__kasan_check_read", + "__kasan_check_write", /* KASAN in-line */ "__asan_report_load_n_noabort", "__asan_report_load1_noabort", From 894c26a1c274b8eafbb4b1dad67e70e51a106061 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Wed, 30 Sep 2020 22:05:42 +0800 Subject: [PATCH 205/265] ACPI: Support Generic Initiator only domains Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory. This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node. Signed-off-by: Jonathan Cameron Signed-off-by: Rafael J. Wysocki --- drivers/acpi/numa/srat.c | 69 +++++++++++++++++++++++++++++++++++++++- drivers/base/node.c | 3 ++ include/linux/nodemask.h | 1 + 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index 2c9a66c203ff..979e76b94e54 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -130,6 +130,36 @@ acpi_table_print_srat_entry(struct acpi_subtable_header *header) } break; + case ACPI_SRAT_TYPE_GENERIC_AFFINITY: + { + struct acpi_srat_generic_affinity *p = + (struct acpi_srat_generic_affinity *)header; + + if (p->device_handle_type == 0) { + /* + * For pci devices this may be the only place they + * are assigned a proximity domain + */ + pr_debug("SRAT Generic Initiator(Seg:%u BDF:%u) in proximity domain %d %s\n", + *(u16 *)(&p->device_handle[0]), + *(u16 *)(&p->device_handle[2]), + p->proximity_domain, + (p->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED) ? + "enabled" : "disabled"); + } else { + /* + * In this case we can rely on the device having a + * proximity domain reference + */ + pr_debug("SRAT Generic Initiator(HID=%.8s UID=%.4s) in proximity domain %d %s\n", + (char *)(&p->device_handle[0]), + (char *)(&p->device_handle[8]), + p->proximity_domain, + (p->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED) ? + "enabled" : "disabled"); + } + } + break; default: pr_warn("Found unsupported SRAT entry (type = 0x%x)\n", header->type); @@ -332,6 +362,41 @@ acpi_parse_gicc_affinity(union acpi_subtable_headers *header, return 0; } +#if defined(CONFIG_X86) || defined(CONFIG_ARM64) +static int __init +acpi_parse_gi_affinity(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_srat_generic_affinity *gi_affinity; + int node; + + gi_affinity = (struct acpi_srat_generic_affinity *)header; + if (!gi_affinity) + return -EINVAL; + acpi_table_print_srat_entry(&header->common); + + if (!(gi_affinity->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED)) + return -EINVAL; + + node = acpi_map_pxm_to_node(gi_affinity->proximity_domain); + if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) { + pr_err("SRAT: Too many proximity domains.\n"); + return -EINVAL; + } + node_set(node, numa_nodes_parsed); + node_set_state(node, N_GENERIC_INITIATOR); + + return 0; +} +#else +static int __init +acpi_parse_gi_affinity(union acpi_subtable_headers *header, + const unsigned long end) +{ + return 0; +} +#endif /* defined(CONFIG_X86) || defined (CONFIG_ARM64) */ + static int __initdata parsed_numa_memblks; static int __init @@ -385,7 +450,7 @@ int __init acpi_numa_init(void) /* SRAT: System Resource Affinity Table */ if (!acpi_table_parse(ACPI_SIG_SRAT, acpi_parse_srat)) { - struct acpi_subtable_proc srat_proc[3]; + struct acpi_subtable_proc srat_proc[4]; memset(srat_proc, 0, sizeof(srat_proc)); srat_proc[0].id = ACPI_SRAT_TYPE_CPU_AFFINITY; @@ -394,6 +459,8 @@ int __init acpi_numa_init(void) srat_proc[1].handler = acpi_parse_x2apic_affinity; srat_proc[2].id = ACPI_SRAT_TYPE_GICC_AFFINITY; srat_proc[2].handler = acpi_parse_gicc_affinity; + srat_proc[3].id = ACPI_SRAT_TYPE_GENERIC_AFFINITY; + srat_proc[3].handler = acpi_parse_gi_affinity; acpi_table_parse_entries_array(ACPI_SIG_SRAT, sizeof(struct acpi_table_srat), diff --git a/drivers/base/node.c b/drivers/base/node.c index 508b80f6329b..53383f1f683c 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -980,6 +980,8 @@ static struct node_attr node_state_attr[] = { #endif [N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY), [N_CPU] = _NODE_ATTR(has_cpu, N_CPU), + [N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator, + N_GENERIC_INITIATOR), }; static struct attribute *node_state_attrs[] = { @@ -991,6 +993,7 @@ static struct attribute *node_state_attrs[] = { #endif &node_state_attr[N_MEMORY].attr.attr, &node_state_attr[N_CPU].attr.attr, + &node_state_attr[N_GENERIC_INITIATOR].attr.attr, NULL }; diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h index 27e7fa36f707..3334ce056335 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h @@ -399,6 +399,7 @@ enum node_states { #endif N_MEMORY, /* The node has memory(regular, high, movable) */ N_CPU, /* The node has one or more cpus */ + N_GENERIC_INITIATOR, /* The node has one or more Generic Initiators */ NR_NODE_STATES }; From 73bf7382debb1a93fef5ff38222c6a3b62dfea44 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Wed, 30 Sep 2020 22:05:43 +0800 Subject: [PATCH 206/265] x86: Support Generic Initiator only proximity domains In common with memoryless domains only register GI domains if the proximity node is not online. If a domain is already a memory containing domain, or a memoryless domain there is nothing to do just because it also contains a Generic Initiator. Signed-off-by: Jonathan Cameron Acked-by: Borislav Petkov Signed-off-by: Rafael J. Wysocki --- arch/x86/include/asm/numa.h | 2 ++ arch/x86/kernel/setup.c | 1 + arch/x86/mm/numa.c | 21 +++++++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/arch/x86/include/asm/numa.h b/arch/x86/include/asm/numa.h index bbfde3d2662f..f631467272a3 100644 --- a/arch/x86/include/asm/numa.h +++ b/arch/x86/include/asm/numa.h @@ -62,12 +62,14 @@ extern void numa_clear_node(int cpu); extern void __init init_cpu_to_node(void); extern void numa_add_cpu(int cpu); extern void numa_remove_cpu(int cpu); +extern void init_gi_nodes(void); #else /* CONFIG_NUMA */ static inline void numa_set_node(int cpu, int node) { } static inline void numa_clear_node(int cpu) { } static inline void init_cpu_to_node(void) { } static inline void numa_add_cpu(int cpu) { } static inline void numa_remove_cpu(int cpu) { } +static inline void init_gi_nodes(void) { } #endif /* CONFIG_NUMA */ #ifdef CONFIG_DEBUG_PER_CPU_MAPS diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 3511736fbc74..9062c146f03a 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -1218,6 +1218,7 @@ void __init setup_arch(char **cmdline_p) prefill_possible_map(); init_cpu_to_node(); + init_gi_nodes(); io_apic_init_mappings(); diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index aa76ec2d359b..22d3e5ade3ae 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c @@ -747,6 +747,27 @@ static void __init init_memory_less_node(int nid) */ } +/* + * A node may exist which has one or more Generic Initiators but no CPUs and no + * memory. + * + * This function must be called after init_cpu_to_node(), to ensure that any + * memoryless CPU nodes have already been brought online, and before the + * node_data[nid] is needed for zone list setup in build_all_zonelists(). + * + * When this function is called, any nodes containing either memory and/or CPUs + * will already be online and there is no need to do anything extra, even if + * they also contain one or more Generic Initiators. + */ +void __init init_gi_nodes(void) +{ + int nid; + + for_each_node_state(nid, N_GENERIC_INITIATOR) + if (!node_online(nid)) + init_memory_less_node(nid); +} + /* * Setup early cpu_to_node. * From 01aabca2fd545554905321029e6c4c5b2fedb345 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Wed, 30 Sep 2020 22:05:44 +0800 Subject: [PATCH 207/265] ACPI: Let ACPI know we support Generic Initiator Affinity Structures Until we tell ACPI that we support generic initiators, it will have to operate in fall back domain mode and all _PXM entries should be on existing non GI domains. This patch sets the relevant OSC bit to make that happen. Signed-off-by: Jonathan Cameron Signed-off-by: Rafael J. Wysocki --- drivers/acpi/bus.c | 4 ++++ include/linux/acpi.h | 1 + 2 files changed, 5 insertions(+) diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index 54002670cb7a..113c661eb848 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -303,7 +303,11 @@ static void acpi_bus_osc_support(void) capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_HOTPLUG_OST_SUPPORT; capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PCLPI_SUPPORT; +#ifdef CONFIG_ARM64 + capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_GENERIC_INITIATOR_SUPPORT; +#endif #ifdef CONFIG_X86 + capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_GENERIC_INITIATOR_SUPPORT; if (boot_cpu_has(X86_FEATURE_HWP)) { capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_CPC_SUPPORT; capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_CPCV2_SUPPORT; diff --git a/include/linux/acpi.h b/include/linux/acpi.h index e9f6cd67943e..edbf3c4116b4 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -545,6 +545,7 @@ acpi_status acpi_run_osc(acpi_handle handle, struct acpi_osc_context *context); #define OSC_SB_PCLPI_SUPPORT 0x00000080 #define OSC_SB_OSLPI_SUPPORT 0x00000100 #define OSC_SB_CPC_DIVERSE_HIGH_SUPPORT 0x00001000 +#define OSC_SB_GENERIC_INITIATOR_SUPPORT 0x00002000 extern bool osc_sb_apei_support_acked; extern bool osc_pc_lpi_support_confirmed; From 2c5b9bde95c96942f2873cea6ef383c02800e4a8 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Wed, 30 Sep 2020 22:05:45 +0800 Subject: [PATCH 208/265] ACPI: HMAT: Fix handling of changes from ACPI 6.2 to ACPI 6.3 In ACPI 6.3, the Memory Proximity Domain Attributes Structure changed substantially. One of those changes was that the flag for "Memory Proximity Domain field is valid" was deprecated. This was because the field "Proximity Domain for the Memory" became a required field and hence having a validity flag makes no sense. So the correct logic is to always assume the field is there. Current code assumes it never is. Signed-off-by: Jonathan Cameron Signed-off-by: Rafael J. Wysocki --- drivers/acpi/numa/hmat.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index e7add2609c03..5264aee93642 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -424,7 +424,8 @@ static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *heade pr_info("HMAT: Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n", p->flags, p->processor_PD, p->memory_PD); - if (p->flags & ACPI_HMAT_MEMORY_PD_VALID && hmat_revision == 1) { + if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) || + hmat_revision > 1) { target = find_mem_target(p->memory_PD); if (!target) { pr_debug("HMAT: Memory Domain missing from SRAT\n"); From b9fffe47212c70114f1d91e773ce5ecff8936ef5 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Wed, 30 Sep 2020 22:05:46 +0800 Subject: [PATCH 209/265] node: Add access1 class to represent CPU to memory characteristics New access1 class is nearly the same as access0, but always provides characteristics for CPUs to memory. The existing access0 class provides characteristics to nearest or direct connnect initiator which may be a Generic Initiator such as a GPU or network adapter. This new class allows thread placement on CPUs to be performed so as to give optimal access characteristics to memory, even if that memory is for example attached to a GPU or similar and only accessible to the CPU via an appropriate bus. Suggested-by: Dan Willaims Signed-off-by: Jonathan Cameron Signed-off-by: Rafael J. Wysocki --- drivers/acpi/numa/hmat.c | 88 +++++++++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 19 deletions(-) diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index 5264aee93642..6ea95b367a31 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -56,7 +56,7 @@ struct memory_target { unsigned int memory_pxm; unsigned int processor_pxm; struct resource memregions; - struct node_hmem_attrs hmem_attrs; + struct node_hmem_attrs hmem_attrs[2]; struct list_head caches; struct node_cache_attrs cache_attrs; bool registered; @@ -65,6 +65,7 @@ struct memory_target { struct memory_initiator { struct list_head node; unsigned int processor_pxm; + bool has_cpu; }; struct memory_locality { @@ -108,6 +109,7 @@ static __init void alloc_memory_initiator(unsigned int cpu_pxm) return; initiator->processor_pxm = cpu_pxm; + initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU); list_add_tail(&initiator->node, &initiators); } @@ -215,28 +217,28 @@ static u32 hmat_normalize(u16 entry, u64 base, u8 type) } static void hmat_update_target_access(struct memory_target *target, - u8 type, u32 value) + u8 type, u32 value, int access) { switch (type) { case ACPI_HMAT_ACCESS_LATENCY: - target->hmem_attrs.read_latency = value; - target->hmem_attrs.write_latency = value; + target->hmem_attrs[access].read_latency = value; + target->hmem_attrs[access].write_latency = value; break; case ACPI_HMAT_READ_LATENCY: - target->hmem_attrs.read_latency = value; + target->hmem_attrs[access].read_latency = value; break; case ACPI_HMAT_WRITE_LATENCY: - target->hmem_attrs.write_latency = value; + target->hmem_attrs[access].write_latency = value; break; case ACPI_HMAT_ACCESS_BANDWIDTH: - target->hmem_attrs.read_bandwidth = value; - target->hmem_attrs.write_bandwidth = value; + target->hmem_attrs[access].read_bandwidth = value; + target->hmem_attrs[access].write_bandwidth = value; break; case ACPI_HMAT_READ_BANDWIDTH: - target->hmem_attrs.read_bandwidth = value; + target->hmem_attrs[access].read_bandwidth = value; break; case ACPI_HMAT_WRITE_BANDWIDTH: - target->hmem_attrs.write_bandwidth = value; + target->hmem_attrs[access].write_bandwidth = value; break; default: break; @@ -329,8 +331,12 @@ static __init int hmat_parse_locality(union acpi_subtable_headers *header, if (mem_hier == ACPI_HMAT_MEMORY) { target = find_mem_target(targs[targ]); - if (target && target->processor_pxm == inits[init]) - hmat_update_target_access(target, type, value); + if (target && target->processor_pxm == inits[init]) { + hmat_update_target_access(target, type, value, 0); + /* If the node has a CPU, update access 1 */ + if (node_state(pxm_to_node(inits[init]), N_CPU)) + hmat_update_target_access(target, type, value, 1); + } } } } @@ -567,6 +573,7 @@ static void hmat_register_target_initiators(struct memory_target *target) unsigned int mem_nid, cpu_nid; struct memory_locality *loc = NULL; u32 best = 0; + bool access0done = false; int i; mem_nid = pxm_to_node(target->memory_pxm); @@ -578,7 +585,11 @@ static void hmat_register_target_initiators(struct memory_target *target) if (target->processor_pxm != PXM_INVAL) { cpu_nid = pxm_to_node(target->processor_pxm); register_memory_node_under_compute_node(mem_nid, cpu_nid, 0); - return; + access0done = true; + if (node_state(cpu_nid, N_CPU)) { + register_memory_node_under_compute_node(mem_nid, cpu_nid, 1); + return; + } } if (list_empty(&localities)) @@ -592,6 +603,41 @@ static void hmat_register_target_initiators(struct memory_target *target) */ bitmap_zero(p_nodes, MAX_NUMNODES); list_sort(p_nodes, &initiators, initiator_cmp); + if (!access0done) { + for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { + loc = localities_types[i]; + if (!loc) + continue; + + best = 0; + list_for_each_entry(initiator, &initiators, node) { + u32 value; + + if (!test_bit(initiator->processor_pxm, p_nodes)) + continue; + + value = hmat_initiator_perf(target, initiator, + loc->hmat_loc); + if (hmat_update_best(loc->hmat_loc->data_type, value, &best)) + bitmap_clear(p_nodes, 0, initiator->processor_pxm); + if (value != best) + clear_bit(initiator->processor_pxm, p_nodes); + } + if (best) + hmat_update_target_access(target, loc->hmat_loc->data_type, + best, 0); + } + + for_each_set_bit(i, p_nodes, MAX_NUMNODES) { + cpu_nid = pxm_to_node(i); + register_memory_node_under_compute_node(mem_nid, cpu_nid, 0); + } + } + + /* Access 1 ignores Generic Initiators */ + bitmap_zero(p_nodes, MAX_NUMNODES); + list_sort(p_nodes, &initiators, initiator_cmp); + best = 0; for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { loc = localities_types[i]; if (!loc) @@ -601,6 +647,10 @@ static void hmat_register_target_initiators(struct memory_target *target) list_for_each_entry(initiator, &initiators, node) { u32 value; + if (!initiator->has_cpu) { + clear_bit(initiator->processor_pxm, p_nodes); + continue; + } if (!test_bit(initiator->processor_pxm, p_nodes)) continue; @@ -611,12 +661,11 @@ static void hmat_register_target_initiators(struct memory_target *target) clear_bit(initiator->processor_pxm, p_nodes); } if (best) - hmat_update_target_access(target, loc->hmat_loc->data_type, best); + hmat_update_target_access(target, loc->hmat_loc->data_type, best, 1); } - for_each_set_bit(i, p_nodes, MAX_NUMNODES) { cpu_nid = pxm_to_node(i); - register_memory_node_under_compute_node(mem_nid, cpu_nid, 0); + register_memory_node_under_compute_node(mem_nid, cpu_nid, 1); } } @@ -629,10 +678,10 @@ static void hmat_register_target_cache(struct memory_target *target) node_add_cache(mem_nid, &tcache->cache_attrs); } -static void hmat_register_target_perf(struct memory_target *target) +static void hmat_register_target_perf(struct memory_target *target, int access) { unsigned mem_nid = pxm_to_node(target->memory_pxm); - node_set_perf_attrs(mem_nid, &target->hmem_attrs, 0); + node_set_perf_attrs(mem_nid, &target->hmem_attrs[access], access); } static void hmat_register_target_device(struct memory_target *target, @@ -734,7 +783,8 @@ static void hmat_register_target(struct memory_target *target) if (!target->registered) { hmat_register_target_initiators(target); hmat_register_target_cache(target); - hmat_register_target_perf(target); + hmat_register_target_perf(target, 0); + hmat_register_target_perf(target, 1); target->registered = true; } mutex_unlock(&target_lock); From dc9e7860df91625fef04d7016182653e8f678f05 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Wed, 30 Sep 2020 22:05:47 +0800 Subject: [PATCH 210/265] docs: mm: numaperf.rst Add brief description for access class 1. Try to make minimal changes to the document which already describes access class 0 in a generic fashion (including IO initiatiors that are not CPUs). Signed-off-by: Jonathan Cameron Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/mm/numaperf.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Documentation/admin-guide/mm/numaperf.rst b/Documentation/admin-guide/mm/numaperf.rst index 4d69ef1de830..86f2a3c4b638 100644 --- a/Documentation/admin-guide/mm/numaperf.rst +++ b/Documentation/admin-guide/mm/numaperf.rst @@ -56,6 +56,11 @@ nodes' access characteristics share the same performance relative to other linked initiator nodes. Each target within an initiator's access class, though, do not necessarily perform the same as each other. +The access class "1" is used to allow differentiation between initiators +that are CPUs and hence suitable for generic task scheduling, and +IO initiators such as GPUs and NICs. Unlike access class 0, only +nodes containing CPUs are considered. + ================ NUMA Performance ================ @@ -88,6 +93,9 @@ The latency attributes are provided in nanoseconds. The values reported here correspond to the rated latency and bandwidth for the platform. +Access class 1 takes the same form but only includes values for CPU to +memory activity. + ========== NUMA Cache ========== From 1e0cb59d5f0a41d1a2b7a1e7d45024fbac62d6e7 Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 30 Sep 2020 13:43:50 +0100 Subject: [PATCH 211/265] ACPI: Make acpi_evaluate_dsm() prototype consistent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When compiling a driver which includes both include/linux/acpi.h and include/acpi/acpi_bus.h for when CONFIG_ACPI=n for i386, I get this: /include/acpi/acpi_bus.h:53:20: error: conflicting types for ‘acpi_evaluate_dsm’ union acpi_object *acpi_evaluate_dsm(acpi_handle handle, const guid_t *guid, ^~~~~~~~~~~~~~~~~ In file included from drivers/scsi/hisi_sas/hisi_sas.h:10:0, from drivers/scsi/hisi_sas/hisi_sas_main.c:7: ./include/linux/acpi.h:866:34: note: previous definition of ‘acpi_evaluate_dsm’ was here static inline union acpi_object *acpi_evaluate_dsm(acpi_handle handle, ^~~~~~~~~~~~~~~~~ Fix by making prototype in include/linux/acpi.h consistent. Signed-off-by: John Garry [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- include/linux/acpi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 7ce2235f99f9..49dbb105084f 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -865,7 +865,7 @@ static inline bool acpi_driver_match_device(struct device *dev, static inline union acpi_object *acpi_evaluate_dsm(acpi_handle handle, const guid_t *guid, - int rev, int func, + u64 rev, u64 func, union acpi_object *argv4) { return NULL; From 7cecb47f55e00282f972a1e0b09136c8cd938221 Mon Sep 17 00:00:00 2001 From: Ben Hutchings Date: Sun, 27 Sep 2020 22:50:42 +0100 Subject: [PATCH 212/265] ACPI / extlog: Check for RDMSR failure extlog_init() uses rdmsrl() to read an MSR, which on older CPUs provokes a error message at boot: unchecked MSR access error: RDMSR from 0x179 at rIP: 0xcd047307 (native_read_msr+0x7/0x40) Use rdmsrl_safe() instead, and return -ENODEV if it fails. Reported-by: jim@photojim.ca References: https://bugs.debian.org/971058 Cc: All applicable Signed-off-by: Ben Hutchings Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_extlog.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/acpi/acpi_extlog.c b/drivers/acpi/acpi_extlog.c index f138e12b7b82..72f1fb77abcd 100644 --- a/drivers/acpi/acpi_extlog.c +++ b/drivers/acpi/acpi_extlog.c @@ -222,9 +222,9 @@ static int __init extlog_init(void) u64 cap; int rc; - rdmsrl(MSR_IA32_MCG_CAP, cap); - - if (!(cap & MCG_ELOG_P) || !extlog_get_l1addr()) + if (rdmsrl_safe(MSR_IA32_MCG_CAP, &cap) || + !(cap & MCG_ELOG_P) || + !extlog_get_l1addr()) return -ENODEV; rc = -EINVAL; From c18483a8ed30712630bc91bb9ea035cc9e1dfef0 Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Sun, 27 Sep 2020 17:55:49 +0800 Subject: [PATCH 213/265] ACPI: memhotplug: Remove 'state' from struct acpi_memory_device After commit 315bbae9c5cb ("ACPI / memhotplug: deal with eject request in hotplug queue"), the memory device state, which is defined in struct acpi_memory_device, is not actually useful, so remove it along with symbols related to it. Signed-off-by: Hanjun Guo [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_memhotplug.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/drivers/acpi/acpi_memhotplug.c b/drivers/acpi/acpi_memhotplug.c index e294f44a7850..468ebb706913 100644 --- a/drivers/acpi/acpi_memhotplug.c +++ b/drivers/acpi/acpi_memhotplug.c @@ -36,11 +36,6 @@ static const struct acpi_device_id memory_device_ids[] = { #ifdef CONFIG_ACPI_HOTPLUG_MEMORY -/* Memory Device States */ -#define MEMORY_INVALID_STATE 0 -#define MEMORY_POWER_ON_STATE 1 -#define MEMORY_POWER_OFF_STATE 2 - static int acpi_memory_device_add(struct acpi_device *device, const struct acpi_device_id *not_used); static void acpi_memory_device_remove(struct acpi_device *device); @@ -64,8 +59,7 @@ struct acpi_memory_info { }; struct acpi_memory_device { - struct acpi_device * device; - unsigned int state; /* State of the memory device */ + struct acpi_device *device; struct list_head res_list; }; @@ -233,7 +227,6 @@ static int acpi_memory_enable_device(struct acpi_memory_device *mem_device) } if (!num_enabled) { dev_err(&mem_device->device->dev, "add_memory failed\n"); - mem_device->state = MEMORY_INVALID_STATE; return -EINVAL; } /* @@ -304,9 +297,6 @@ static int acpi_memory_device_add(struct acpi_device *device, return result; } - /* Set the device state */ - mem_device->state = MEMORY_POWER_ON_STATE; - result = acpi_memory_check_device(mem_device); if (result) { acpi_memory_device_free(mem_device); From 05de068614ac4b5bc37a0f53df713cf4475ae924 Mon Sep 17 00:00:00 2001 From: Tian Tao Date: Sun, 27 Sep 2020 09:14:28 +0800 Subject: [PATCH 214/265] ACPI: scan: Replace ACPI_DEBUG_PRINT() with pr_debug() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the following W=1 kernel build warning(s): drivers/acpi/scan.c: In function ‘acpi_bus_get_wakeup_device_flags’: drivers/acpi/scan.c:902:43: warning: suggest braces around empty body in an ‘if’ statement [-Wempty-body] by using pr_debug() to instead of the ACPI_DEBUG_PRINT() macro, which should only be used by ACPICA code, to print a debug message. Signed-off-by: Tian Tao [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/scan.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index 2142f1554761..684c726828e1 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -898,8 +898,7 @@ static void acpi_bus_get_wakeup_device_flags(struct acpi_device *device) */ err = acpi_device_sleep_wake(device, 0, 0, 0); if (err) - ACPI_DEBUG_PRINT((ACPI_DB_INFO, - "error in _DSW or _PSW evaluation\n")); + pr_debug("error in _DSW or _PSW evaluation\n"); } static void acpi_bus_init_power_state(struct acpi_device *device, int state) From 49f618e1b669ef0e26a8d8d7f8fafc7b8fd31531 Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Thu, 24 Sep 2020 13:04:47 +0200 Subject: [PATCH 215/265] PM: domains: Rename power state enums for genpd To clarify the code a bit, let's rename GPD_STATE_ACTIVE into GENPD_STATE_ON and GPD_STATE_POWER_OFF to GENPD_STATE_OFF. Signed-off-by: Ulf Hansson [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/base/power/domain.c | 33 ++++++++++++++++----------------- include/linux/pm_domain.h | 4 ++-- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 2cb5e04cf86c..23aa2feced77 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -123,7 +123,7 @@ static const struct genpd_lock_ops genpd_spin_ops = { #define genpd_lock_interruptible(p) p->lock_ops->lock_interruptible(p) #define genpd_unlock(p) p->lock_ops->unlock(p) -#define genpd_status_on(genpd) (genpd->status == GPD_STATE_ACTIVE) +#define genpd_status_on(genpd) (genpd->status == GENPD_STATE_ON) #define genpd_is_irq_safe(genpd) (genpd->flags & GENPD_FLAG_IRQ_SAFE) #define genpd_is_always_on(genpd) (genpd->flags & GENPD_FLAG_ALWAYS_ON) #define genpd_is_active_wakeup(genpd) (genpd->flags & GENPD_FLAG_ACTIVE_WAKEUP) @@ -222,7 +222,7 @@ static void genpd_update_accounting(struct generic_pm_domain *genpd) * out of off and so update the idle time and vice * versa. */ - if (genpd->status == GPD_STATE_ACTIVE) { + if (genpd->status == GENPD_STATE_ON) { int state_idx = genpd->state_idx; genpd->states[state_idx].idle_time = @@ -563,7 +563,7 @@ static int genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, return ret; } - genpd->status = GPD_STATE_POWER_OFF; + genpd->status = GENPD_STATE_OFF; genpd_update_accounting(genpd); list_for_each_entry(link, &genpd->child_links, child_node) { @@ -616,7 +616,7 @@ static int genpd_power_on(struct generic_pm_domain *genpd, unsigned int depth) if (ret) goto err; - genpd->status = GPD_STATE_ACTIVE; + genpd->status = GENPD_STATE_ON; genpd_update_accounting(genpd); return 0; @@ -961,7 +961,7 @@ static void genpd_sync_power_off(struct generic_pm_domain *genpd, bool use_lock, if (_genpd_power_off(genpd, false)) return; - genpd->status = GPD_STATE_POWER_OFF; + genpd->status = GENPD_STATE_OFF; list_for_each_entry(link, &genpd->child_links, child_node) { genpd_sd_counter_dec(link->parent); @@ -1007,8 +1007,7 @@ static void genpd_sync_power_on(struct generic_pm_domain *genpd, bool use_lock, } _genpd_power_on(genpd, false); - - genpd->status = GPD_STATE_ACTIVE; + genpd->status = GENPD_STATE_ON; } /** @@ -1287,7 +1286,7 @@ static int genpd_restore_noirq(struct device *dev) * so make it appear as powered off to genpd_sync_power_on(), * so that it tries to power it on in case it was really off. */ - genpd->status = GPD_STATE_POWER_OFF; + genpd->status = GENPD_STATE_OFF; genpd_sync_power_on(genpd, true, 0); genpd_unlock(genpd); @@ -1777,7 +1776,7 @@ int pm_genpd_init(struct generic_pm_domain *genpd, genpd->gov = gov; INIT_WORK(&genpd->power_off_work, genpd_power_off_work_fn); atomic_set(&genpd->sd_count, 0); - genpd->status = is_off ? GPD_STATE_POWER_OFF : GPD_STATE_ACTIVE; + genpd->status = is_off ? GENPD_STATE_OFF : GENPD_STATE_ON; genpd->device_count = 0; genpd->max_off_time_ns = -1; genpd->max_off_time_changed = true; @@ -2802,8 +2801,8 @@ static int genpd_summary_one(struct seq_file *s, struct generic_pm_domain *genpd) { static const char * const status_lookup[] = { - [GPD_STATE_ACTIVE] = "on", - [GPD_STATE_POWER_OFF] = "off" + [GENPD_STATE_ON] = "on", + [GENPD_STATE_OFF] = "off" }; struct pm_domain_data *pm_data; const char *kobj_path; @@ -2881,8 +2880,8 @@ static int summary_show(struct seq_file *s, void *data) static int status_show(struct seq_file *s, void *data) { static const char * const status_lookup[] = { - [GPD_STATE_ACTIVE] = "on", - [GPD_STATE_POWER_OFF] = "off" + [GENPD_STATE_ON] = "on", + [GENPD_STATE_OFF] = "off" }; struct generic_pm_domain *genpd = s->private; @@ -2895,7 +2894,7 @@ static int status_show(struct seq_file *s, void *data) if (WARN_ON_ONCE(genpd->status >= ARRAY_SIZE(status_lookup))) goto exit; - if (genpd->status == GPD_STATE_POWER_OFF) + if (genpd->status == GENPD_STATE_OFF) seq_printf(s, "%s-%u\n", status_lookup[genpd->status], genpd->state_idx); else @@ -2938,7 +2937,7 @@ static int idle_states_show(struct seq_file *s, void *data) ktime_t delta = 0; s64 msecs; - if ((genpd->status == GPD_STATE_POWER_OFF) && + if ((genpd->status == GENPD_STATE_OFF) && (genpd->state_idx == i)) delta = ktime_sub(ktime_get(), genpd->accounting_time); @@ -2961,7 +2960,7 @@ static int active_time_show(struct seq_file *s, void *data) if (ret) return -ERESTARTSYS; - if (genpd->status == GPD_STATE_ACTIVE) + if (genpd->status == GENPD_STATE_ON) delta = ktime_sub(ktime_get(), genpd->accounting_time); seq_printf(s, "%lld ms\n", ktime_to_ms( @@ -2984,7 +2983,7 @@ static int total_idle_time_show(struct seq_file *s, void *data) for (i = 0; i < genpd->state_count; i++) { - if ((genpd->status == GPD_STATE_POWER_OFF) && + if ((genpd->status == GENPD_STATE_OFF) && (genpd->state_idx == i)) delta = ktime_sub(ktime_get(), genpd->accounting_time); diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index ee11502a575b..66f3c5d64d81 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -64,8 +64,8 @@ #define GENPD_FLAG_RPM_ALWAYS_ON (1U << 5) enum gpd_status { - GPD_STATE_ACTIVE = 0, /* PM domain is active */ - GPD_STATE_POWER_OFF, /* PM domain is off */ + GENPD_STATE_ON = 0, /* PM domain is on */ + GENPD_STATE_OFF, /* PM domain is off */ }; struct dev_power_governor { From f63816e43d90442684cd2fd74f602cf3c5fae69c Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Thu, 24 Sep 2020 13:04:48 +0200 Subject: [PATCH 216/265] PM: domains: Allow to abort power off when no ->power_off() callback In genpd_power_off() we may decide to abort the power off of the PM domain, even beyond the point when the governor would accept it. The abort is done if it turns out that a child domain has been requested to be powered on, which means it's waiting for the lock of the parent to be released. However, the abort is currently only considered if the genpd in question has a ->power_off() callback assigned. This is unnecessary limiting, especially if the genpd would have a parent of its own. Let's remove the limitation and make the behaviour consistent. Signed-off-by: Ulf Hansson [ rjw: Subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/base/power/domain.c | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 23aa2feced77..0198af358503 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -497,6 +497,7 @@ static int genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, struct pm_domain_data *pdd; struct gpd_link *link; unsigned int not_suspended = 0; + int ret; /* * Do not try to power off the domain in the following situations: @@ -544,24 +545,13 @@ static int genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, if (!genpd->gov) genpd->state_idx = 0; - if (genpd->power_off) { - int ret; + /* Don't power off, if a child domain is waiting to power on. */ + if (atomic_read(&genpd->sd_count) > 0) + return -EBUSY; - if (atomic_read(&genpd->sd_count) > 0) - return -EBUSY; - - /* - * If sd_count > 0 at this point, one of the subdomains hasn't - * managed to call genpd_power_on() for the parent yet after - * incrementing it. In that case genpd_power_on() will wait - * for us to drop the lock, so we can call .power_off() and let - * the genpd_power_on() restore power for us (this shouldn't - * happen very often). - */ - ret = _genpd_power_off(genpd, true); - if (ret) - return ret; - } + ret = _genpd_power_off(genpd, true); + if (ret) + return ret; genpd->status = GENPD_STATE_OFF; genpd_update_accounting(genpd); From 05c36e5adf2beae64d118f49fcaafbe8ce373db1 Mon Sep 17 00:00:00 2001 From: Thomas Renninger Date: Fri, 25 Sep 2020 17:21:04 +0200 Subject: [PATCH 217/265] tools/power/acpi: Serialize Makefile Before this patch you get tools/power/acpi/Makefile.rules included in parallel trying to copy KERNEL_INCLUDE multiple times: make -j20 acpi DESCEND power/acpi DESCEND tools/acpidbg DESCEND tools/acpidump DESCEND tools/ec MKDIR include MKDIR include MKDIR include CP include CP include cp: cannot create directory '/home/abuild/rpmbuild/BUILD/linux-5.7.7+git20200917.10b82d517648/tools/power/acpi/include/acpi': File exists make[2]: *** [../../Makefile.rules:20: /home/abuild/rpmbuild/BUILD/linux-5.7.7+git20200917.10b82d517648/tools/power/acpi/include] Error 1 make[1]: *** [Makefile:16: acpidbg] Error 2 make[1]: *** Waiting for unfinished jobs.... with this patch each subdirectory will be processed serialized: DESCEND power/acpi DESCEND tools/acpidbg MKDIR include CP include CC tools/acpidbg/acpidbg.o LD acpidbg STRIP acpidbg DESCEND tools/acpidump CC tools/acpidump/apdump.o ... LD acpidump STRIP acpidump DESCEND tools/ec CC tools/ec/ec_access.o LD ec STRIP ec Signed-off-by: Thomas Renninger Signed-off-by: Rafael J. Wysocki --- tools/power/acpi/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/power/acpi/Makefile b/tools/power/acpi/Makefile index ebd3e1a1c28e..a249c50ebf55 100644 --- a/tools/power/acpi/Makefile +++ b/tools/power/acpi/Makefile @@ -7,6 +7,8 @@ include ../../scripts/Makefile.include +.NOTPARALLEL: + all: acpidbg acpidump ec clean: acpidbg_clean acpidump_clean ec_clean install: acpidbg_install acpidump_install ec_install From 779055842da5b2e508f3ccf9a8153cb1f704f566 Mon Sep 17 00:00:00 2001 From: Souptick Joarder Date: Sun, 6 Sep 2020 12:21:53 +0530 Subject: [PATCH 218/265] xen/gntdev.c: Mark pages as dirty There seems to be a bug in the original code when gntdev_get_page() is called with writeable=true then the page needs to be marked dirty before being put. To address this, a bool writeable is added in gnt_dev_copy_batch, set it in gntdev_grant_copy_seg() (and drop `writeable` argument to gntdev_get_page()) and then, based on batch->writeable, use set_page_dirty_lock(). Fixes: a4cdb556cae0 (xen/gntdev: add ioctl for grant copy) Suggested-by: Boris Ostrovsky Signed-off-by: Souptick Joarder Cc: John Hubbard Cc: Boris Ostrovsky Cc: Juergen Gross Cc: David Vrabel Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/1599375114-32360-1-git-send-email-jrdr.linux@gmail.com Reviewed-by: Boris Ostrovsky Signed-off-by: Boris Ostrovsky --- drivers/xen/gntdev.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index 64a9025a87be..1f32db7b72b2 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -720,17 +720,18 @@ struct gntdev_copy_batch { s16 __user *status[GNTDEV_COPY_BATCH]; unsigned int nr_ops; unsigned int nr_pages; + bool writeable; }; static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt, - bool writeable, unsigned long *gfn) + unsigned long *gfn) { unsigned long addr = (unsigned long)virt; struct page *page; unsigned long xen_pfn; int ret; - ret = get_user_pages_fast(addr, 1, writeable ? FOLL_WRITE : 0, &page); + ret = get_user_pages_fast(addr, 1, batch->writeable ? FOLL_WRITE : 0, &page); if (ret < 0) return ret; @@ -746,9 +747,13 @@ static void gntdev_put_pages(struct gntdev_copy_batch *batch) { unsigned int i; - for (i = 0; i < batch->nr_pages; i++) + for (i = 0; i < batch->nr_pages; i++) { + if (batch->writeable && !PageDirty(batch->pages[i])) + set_page_dirty_lock(batch->pages[i]); put_page(batch->pages[i]); + } batch->nr_pages = 0; + batch->writeable = false; } static int gntdev_copy(struct gntdev_copy_batch *batch) @@ -837,8 +842,9 @@ static int gntdev_grant_copy_seg(struct gntdev_copy_batch *batch, virt = seg->source.virt + copied; off = (unsigned long)virt & ~XEN_PAGE_MASK; len = min(len, (size_t)XEN_PAGE_SIZE - off); + batch->writeable = false; - ret = gntdev_get_page(batch, virt, false, &gfn); + ret = gntdev_get_page(batch, virt, &gfn); if (ret < 0) return ret; @@ -856,8 +862,9 @@ static int gntdev_grant_copy_seg(struct gntdev_copy_batch *batch, virt = seg->dest.virt + copied; off = (unsigned long)virt & ~XEN_PAGE_MASK; len = min(len, (size_t)XEN_PAGE_SIZE - off); + batch->writeable = true; - ret = gntdev_get_page(batch, virt, true, &gfn); + ret = gntdev_get_page(batch, virt, &gfn); if (ret < 0) return ret; From d6bbc2ff68fd22dd972c684921ea348ed7ffcbb2 Mon Sep 17 00:00:00 2001 From: Souptick Joarder Date: Sun, 6 Sep 2020 12:21:54 +0530 Subject: [PATCH 219/265] xen/gntdev.c: Convert get_user_pages*() to pin_user_pages*() In 2019, we introduced pin_user_pages*() and now we are converting get_user_pages*() to the new API as appropriate. [1] & [2] could be referred for more information. This is case 5 as per document [1]. [1] Documentation/core-api/pin_user_pages.rst [2] "Explicit pinning of user-space pages": https://lwn.net/Articles/807108/ Signed-off-by: Souptick Joarder Cc: John Hubbard Cc: Boris Ostrovsky Cc: Juergen Gross Cc: David Vrabel Link: https://lore.kernel.org/r/1599375114-32360-2-git-send-email-jrdr.linux@gmail.com Reviewed-by: Boris Ostrovsky Signed-off-by: Boris Ostrovsky --- drivers/xen/gntdev.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index 1f32db7b72b2..a36b71286bcf 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -731,7 +731,7 @@ static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt, unsigned long xen_pfn; int ret; - ret = get_user_pages_fast(addr, 1, batch->writeable ? FOLL_WRITE : 0, &page); + ret = pin_user_pages_fast(addr, 1, batch->writeable ? FOLL_WRITE : 0, &page); if (ret < 0) return ret; @@ -745,13 +745,7 @@ static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt, static void gntdev_put_pages(struct gntdev_copy_batch *batch) { - unsigned int i; - - for (i = 0; i < batch->nr_pages; i++) { - if (batch->writeable && !PageDirty(batch->pages[i])) - set_page_dirty_lock(batch->pages[i]); - put_page(batch->pages[i]); - } + unpin_user_pages_dirty_lock(batch->pages, batch->nr_pages, batch->writeable); batch->nr_pages = 0; batch->writeable = false; } From bba91dd6d70b7aa239280dc095a461bc064c98a5 Mon Sep 17 00:00:00 2001 From: Jing Xiangfeng Date: Sat, 19 Sep 2020 11:17:02 +0800 Subject: [PATCH 220/265] xen: remove redundant initialization of variable ret After commit 9f51c05dc41a ("pvcalls-front: Avoid get_free_pages(GFP_KERNEL) under spinlock"), the variable ret is being initialized with '-ENOMEM' that is meaningless. So remove it. Signed-off-by: Jing Xiangfeng Link: https://lore.kernel.org/r/20200919031702.32192-1-jingxiangfeng@huawei.com Reviewed-by: Juergen Gross Signed-off-by: Boris Ostrovsky --- drivers/xen/pvcalls-front.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/xen/pvcalls-front.c b/drivers/xen/pvcalls-front.c index 72d725a0ab5c..7984645b5956 100644 --- a/drivers/xen/pvcalls-front.c +++ b/drivers/xen/pvcalls-front.c @@ -371,7 +371,7 @@ out: static int create_active(struct sock_mapping *map, evtchn_port_t *evtchn) { void *bytes; - int ret = -ENOMEM, irq = -1, i; + int ret, irq = -1, i; *evtchn = 0; init_waitqueue_head(&map->active.inflight_conn_req); From f88af7229f6f22ce7313c11a0bbb9c88317b7c5e Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Thu, 24 Sep 2020 16:49:55 -0700 Subject: [PATCH 221/265] xen/arm: do not setup the runstate info page if kpti is enabled The VCPUOP_register_runstate_memory_area hypercall takes a virtual address of a buffer as a parameter. The semantics of the hypercall are such that the virtual address should always be valid. When KPTI is enabled and we are running userspace code, the virtual address is not valid, thus, Linux is violating the semantics of VCPUOP_register_runstate_memory_area. Do not call VCPUOP_register_runstate_memory_area when KPTI is enabled. Signed-off-by: Stefano Stabellini CC: Bertrand Marquis CC: boris.ostrovsky@oracle.com CC: jgross@suse.com Link: https://lore.kernel.org/r/20200924234955.15455-1-sstabellini@kernel.org Reviewed-by: Bertrand Marquis Signed-off-by: Boris Ostrovsky --- arch/arm/include/asm/xen/page.h | 5 +++++ arch/arm/xen/enlighten.c | 6 ++++-- arch/arm64/include/asm/xen/page.h | 6 ++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/arch/arm/include/asm/xen/page.h b/arch/arm/include/asm/xen/page.h index 31bbc803cecb..dc7f6e91aafa 100644 --- a/arch/arm/include/asm/xen/page.h +++ b/arch/arm/include/asm/xen/page.h @@ -1 +1,6 @@ #include + +static inline bool xen_kernel_unmapped_at_usr(void) +{ + return false; +} diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c index e93145d72c26..ea76562af1e9 100644 --- a/arch/arm/xen/enlighten.c +++ b/arch/arm/xen/enlighten.c @@ -158,7 +158,8 @@ static int xen_starting_cpu(unsigned int cpu) BUG_ON(err); per_cpu(xen_vcpu, cpu) = vcpup; - xen_setup_runstate_info(cpu); + if (!xen_kernel_unmapped_at_usr()) + xen_setup_runstate_info(cpu); after_register_vcpu_info: enable_percpu_irq(xen_events_irq, 0); @@ -387,7 +388,8 @@ static int __init xen_guest_init(void) return -EINVAL; } - xen_time_setup_guest(); + if (!xen_kernel_unmapped_at_usr()) + xen_time_setup_guest(); if (xen_initial_domain()) pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier); diff --git a/arch/arm64/include/asm/xen/page.h b/arch/arm64/include/asm/xen/page.h index 31bbc803cecb..dffdc773221b 100644 --- a/arch/arm64/include/asm/xen/page.h +++ b/arch/arm64/include/asm/xen/page.h @@ -1 +1,7 @@ #include +#include + +static inline bool xen_kernel_unmapped_at_usr(void) +{ + return arm64_kernel_unmapped_at_el0(); +} From d759af38572f97321112a0852353613d18126038 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Fri, 25 Sep 2020 16:07:51 +0200 Subject: [PATCH 222/265] x86/xen: disable Firmware First mode for correctable memory errors When running as Xen dom0 the kernel isn't responsible for selecting the error handling mode, this should be handled by the hypervisor. So disable setting FF mode when running as Xen pv guest. Not doing so might result in boot splats like: [ 7.509696] HEST: Enabling Firmware First mode for corrected errors. [ 7.510382] mce: [Firmware Bug]: Ignoring request to disable invalid MCA bank 2. [ 7.510383] mce: [Firmware Bug]: Ignoring request to disable invalid MCA bank 3. [ 7.510384] mce: [Firmware Bug]: Ignoring request to disable invalid MCA bank 4. [ 7.510384] mce: [Firmware Bug]: Ignoring request to disable invalid MCA bank 5. [ 7.510385] mce: [Firmware Bug]: Ignoring request to disable invalid MCA bank 6. [ 7.510386] mce: [Firmware Bug]: Ignoring request to disable invalid MCA bank 7. [ 7.510386] mce: [Firmware Bug]: Ignoring request to disable invalid MCA bank 8. Reason is that the HEST ACPI table contains the real number of MCA banks, while the hypervisor is emulating only 2 banks for guests. Cc: stable@vger.kernel.org Signed-off-by: Juergen Gross Reviewed-by: Boris Ostrovsky Link: https://lore.kernel.org/r/20200925140751.31381-1-jgross@suse.com Signed-off-by: Boris Ostrovsky --- arch/x86/xen/enlighten_pv.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 22e741e0b10c..351ac1a9a119 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -1376,6 +1376,15 @@ asmlinkage __visible void __init xen_start_kernel(void) x86_init.mpparse.get_smp_config = x86_init_uint_noop; xen_boot_params_init_edd(); + +#ifdef CONFIG_ACPI + /* + * Disable selecting "Firmware First mode" for correctable + * memory errors, as this is the duty of the hypervisor to + * decide. + */ + acpi_disable_cmcff = 1; +#endif } if (!boot_params.screen_info.orig_video_isVGA) From 32118f97f41d26a2447118fa956715cb4bd1bdac Mon Sep 17 00:00:00 2001 From: Hui Su Date: Mon, 28 Sep 2020 01:28:36 +0800 Subject: [PATCH 223/265] x86/xen: Fix typo in xen_pagetable_p2m_free() Fix typo in xen_pagetable_p2m_free(): s/Fortunatly/Fortunately Signed-off-by: Hui Su Link: https://lore.kernel.org/r/20200927172836.GA7423@rlk [boris: reword commit message slightly] Signed-off-by: Boris Ostrovsky --- arch/x86/xen/mmu_pv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index 3273c985d3dd..fbee8f50088b 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -1149,7 +1149,7 @@ static void __init xen_pagetable_p2m_free(void) * We could be in __ka space. * We roundup to the PMD, which means that if anybody at this stage is * using the __ka address of xen_start_info or - * xen_start_info->shared_info they are in going to crash. Fortunatly + * xen_start_info->shared_info they are in going to crash. Fortunately * we have already revectored in xen_setup_kernel_pagetable. */ size = roundup(size, PMD_SIZE); From 21988a8e51479ceffe7b0568b170effabb708dfe Mon Sep 17 00:00:00 2001 From: "dmitry.torokhov@gmail.com" Date: Sun, 4 Oct 2020 22:11:25 -0700 Subject: [PATCH 224/265] ACPI: button: fix handling lid state changes when input device closed The original intent of 84d3f6b76447 was to delay evaluating lid state until all drivers have been loaded, with input device being opened from userspace serving as a signal for this condition. Let's ensure that state updates happen even if userspace closed (or in the future inhibited) input device. Note that if we go through suspend/resume cycle we assume the system has been fully initialized even if LID input device has not been opened yet. This has a side-effect of fixing access to input->users outside of input->mutex protections by the way of eliminating said accesses and using driver private flag. Fixes: 84d3f6b76447 ("ACPI / button: Delay acpi_lid_initialize_state() until first user space open") Signed-off-by: Dmitry Torokhov Reviewed-by: Hans de Goede Cc: 4.15+ # 4.15+ Signed-off-by: Rafael J. Wysocki --- drivers/acpi/button.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c index a4eda7fe50d3..da4b125ab4c3 100644 --- a/drivers/acpi/button.c +++ b/drivers/acpi/button.c @@ -153,6 +153,7 @@ struct acpi_button { int last_state; ktime_t last_time; bool suspended; + bool lid_state_initialized; }; static struct acpi_device *lid_device; @@ -383,6 +384,8 @@ static int acpi_lid_update_state(struct acpi_device *device, static void acpi_lid_initialize_state(struct acpi_device *device) { + struct acpi_button *button = acpi_driver_data(device); + switch (lid_init_state) { case ACPI_BUTTON_LID_INIT_OPEN: (void)acpi_lid_notify_state(device, 1); @@ -394,13 +397,14 @@ static void acpi_lid_initialize_state(struct acpi_device *device) default: break; } + + button->lid_state_initialized = true; } static void acpi_button_notify(struct acpi_device *device, u32 event) { struct acpi_button *button = acpi_driver_data(device); struct input_dev *input; - int users; switch (event) { case ACPI_FIXED_HARDWARE_EVENT: @@ -409,10 +413,7 @@ static void acpi_button_notify(struct acpi_device *device, u32 event) case ACPI_BUTTON_NOTIFY_STATUS: input = button->input; if (button->type == ACPI_BUTTON_TYPE_LID) { - mutex_lock(&button->input->mutex); - users = button->input->users; - mutex_unlock(&button->input->mutex); - if (users) + if (button->lid_state_initialized) acpi_lid_update_state(device, true); } else { int keycode; @@ -457,7 +458,7 @@ static int acpi_button_resume(struct device *dev) struct acpi_button *button = acpi_driver_data(device); button->suspended = false; - if (button->type == ACPI_BUTTON_TYPE_LID && button->input->users) { + if (button->type == ACPI_BUTTON_TYPE_LID) { button->last_state = !!acpi_lid_evaluate_state(device); button->last_time = ktime_get(); acpi_lid_initialize_state(device); From 40c3bd4cfa6fe5756344709b98611a1ddd15bf9d Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 5 Oct 2020 13:26:01 +0530 Subject: [PATCH 225/265] cpufreq: stats: Defer stats update to cpufreq_stats_record_transition() In order to prepare for lock-less stats update, add support to defer any updates to it until cpufreq_stats_record_transition() is called. The stats were updated from two places earlier: - show_time_in_state(): This can be easily deferred, all we need is to calculate the delta duration again in this routine to show the current state's time-in-state. - store_reset(): This is a bit tricky as we need to clear the stats here and avoid races with simultaneous call to cpufreq_stats_record_transition(). Fix that by deferring the reset of the stats (within the code) to the next call to cpufreq_stats_record_transition(), but since we need to keep showing the right stats until that time, we capture the reset time and account for the time since last time reset was called until the time cpufreq_stats_record_transition() update the stats. User space will continue seeing the stats correctly, everything will be 0 after the stats are reset, apart from the time-in-state of the current state, until the time a frequency switch happens. Signed-off-by: Viresh Kumar [ rjw: Minor changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq_stats.c | 75 ++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index 94d959a8e954..498d962ba224 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -22,17 +22,22 @@ struct cpufreq_stats { spinlock_t lock; unsigned int *freq_table; unsigned int *trans_table; + + /* Deferred reset */ + unsigned int reset_pending; + unsigned long long reset_time; }; -static void cpufreq_stats_update(struct cpufreq_stats *stats) +static void cpufreq_stats_update(struct cpufreq_stats *stats, + unsigned long long time) { unsigned long long cur_time = get_jiffies_64(); - stats->time_in_state[stats->last_index] += cur_time - stats->last_time; + stats->time_in_state[stats->last_index] += cur_time - time; stats->last_time = cur_time; } -static void cpufreq_stats_clear_table(struct cpufreq_stats *stats) +static void cpufreq_stats_reset_table(struct cpufreq_stats *stats) { unsigned int count = stats->max_state; @@ -41,42 +46,67 @@ static void cpufreq_stats_clear_table(struct cpufreq_stats *stats) memset(stats->trans_table, 0, count * count * sizeof(int)); stats->last_time = get_jiffies_64(); stats->total_trans = 0; + + /* Adjust for the time elapsed since reset was requested */ + WRITE_ONCE(stats->reset_pending, 0); + cpufreq_stats_update(stats, READ_ONCE(stats->reset_time)); spin_unlock(&stats->lock); } static ssize_t show_total_trans(struct cpufreq_policy *policy, char *buf) { - return sprintf(buf, "%d\n", policy->stats->total_trans); + struct cpufreq_stats *stats = policy->stats; + + if (READ_ONCE(stats->reset_pending)) + return sprintf(buf, "%d\n", 0); + else + return sprintf(buf, "%d\n", stats->total_trans); } cpufreq_freq_attr_ro(total_trans); static ssize_t show_time_in_state(struct cpufreq_policy *policy, char *buf) { struct cpufreq_stats *stats = policy->stats; + bool pending = READ_ONCE(stats->reset_pending); + unsigned long long time; ssize_t len = 0; int i; if (policy->fast_switch_enabled) return 0; - spin_lock(&stats->lock); - cpufreq_stats_update(stats); - spin_unlock(&stats->lock); - for (i = 0; i < stats->state_num; i++) { + if (pending) { + if (i == stats->last_index) + time = get_jiffies_64() - READ_ONCE(stats->reset_time); + else + time = 0; + } else { + time = stats->time_in_state[i]; + if (i == stats->last_index) + time += get_jiffies_64() - stats->last_time; + } + len += sprintf(buf + len, "%u %llu\n", stats->freq_table[i], - (unsigned long long) - jiffies_64_to_clock_t(stats->time_in_state[i])); + jiffies_64_to_clock_t(time)); } return len; } cpufreq_freq_attr_ro(time_in_state); +/* We don't care what is written to the attribute */ static ssize_t store_reset(struct cpufreq_policy *policy, const char *buf, size_t count) { - /* We don't care what is written to the attribute. */ - cpufreq_stats_clear_table(policy->stats); + struct cpufreq_stats *stats = policy->stats; + + /* + * Defer resetting of stats to cpufreq_stats_record_transition() to + * avoid races. + */ + WRITE_ONCE(stats->reset_time, get_jiffies_64()); + WRITE_ONCE(stats->reset_pending, 1); + return count; } cpufreq_freq_attr_wo(reset); @@ -84,8 +114,9 @@ cpufreq_freq_attr_wo(reset); static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf) { struct cpufreq_stats *stats = policy->stats; + bool pending = READ_ONCE(stats->reset_pending); ssize_t len = 0; - int i, j; + int i, j, count; if (policy->fast_switch_enabled) return 0; @@ -113,8 +144,13 @@ static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf) for (j = 0; j < stats->state_num; j++) { if (len >= PAGE_SIZE) break; - len += scnprintf(buf + len, PAGE_SIZE - len, "%9u ", - stats->trans_table[i*stats->max_state+j]); + + if (pending) + count = 0; + else + count = stats->trans_table[i * stats->max_state + j]; + + len += scnprintf(buf + len, PAGE_SIZE - len, "%9u ", count); } if (len >= PAGE_SIZE) break; @@ -228,10 +264,11 @@ void cpufreq_stats_record_transition(struct cpufreq_policy *policy, struct cpufreq_stats *stats = policy->stats; int old_index, new_index; - if (!stats) { - pr_debug("%s: No stats found\n", __func__); + if (!stats) return; - } + + if (unlikely(READ_ONCE(stats->reset_pending))) + cpufreq_stats_reset_table(stats); old_index = stats->last_index; new_index = freq_table_get_index(stats, new_freq); @@ -241,7 +278,7 @@ void cpufreq_stats_record_transition(struct cpufreq_policy *policy, return; spin_lock(&stats->lock); - cpufreq_stats_update(stats); + cpufreq_stats_update(stats, stats->last_time); stats->last_index = new_index; stats->trans_table[old_index * stats->max_state + new_index]++; From 381abb942a392377f614b62c87692f4f2e8aa59b Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 5 Oct 2020 13:26:02 +0530 Subject: [PATCH 226/265] cpufreq: stats: Remove locking The locking isn't required anymore as stats can get updated only from one place at a time. Remove it. Signed-off-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq_stats.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index 498d962ba224..45a067855367 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -19,7 +19,6 @@ struct cpufreq_stats { unsigned int state_num; unsigned int last_index; u64 *time_in_state; - spinlock_t lock; unsigned int *freq_table; unsigned int *trans_table; @@ -41,7 +40,6 @@ static void cpufreq_stats_reset_table(struct cpufreq_stats *stats) { unsigned int count = stats->max_state; - spin_lock(&stats->lock); memset(stats->time_in_state, 0, count * sizeof(u64)); memset(stats->trans_table, 0, count * count * sizeof(int)); stats->last_time = get_jiffies_64(); @@ -50,7 +48,6 @@ static void cpufreq_stats_reset_table(struct cpufreq_stats *stats) /* Adjust for the time elapsed since reset was requested */ WRITE_ONCE(stats->reset_pending, 0); cpufreq_stats_update(stats, READ_ONCE(stats->reset_time)); - spin_unlock(&stats->lock); } static ssize_t show_total_trans(struct cpufreq_policy *policy, char *buf) @@ -244,7 +241,6 @@ void cpufreq_stats_create_table(struct cpufreq_policy *policy) stats->state_num = i; stats->last_time = get_jiffies_64(); stats->last_index = freq_table_get_index(stats, policy->cur); - spin_lock_init(&stats->lock); policy->stats = stats; ret = sysfs_create_group(&policy->kobj, &stats_attr_group); @@ -277,11 +273,9 @@ void cpufreq_stats_record_transition(struct cpufreq_policy *policy, if (old_index == -1 || new_index == -1 || old_index == new_index) return; - spin_lock(&stats->lock); cpufreq_stats_update(stats, stats->last_time); stats->last_index = new_index; stats->trans_table[old_index * stats->max_state + new_index]++; stats->total_trans++; - spin_unlock(&stats->lock); } From 4958b46efb6df20fa3abb3a4d007515485f6af16 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 5 Oct 2020 13:26:03 +0530 Subject: [PATCH 227/265] cpufreq: stats: Mark few conditionals with unlikely() Since this will be part of the scheduler's hotpath in some cases, use unlikely() for few of the obvious conditionals. Signed-off-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq_stats.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index 45a067855367..bba04da3a278 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -260,7 +260,7 @@ void cpufreq_stats_record_transition(struct cpufreq_policy *policy, struct cpufreq_stats *stats = policy->stats; int old_index, new_index; - if (!stats) + if (unlikely(!stats)) return; if (unlikely(READ_ONCE(stats->reset_pending))) @@ -270,7 +270,7 @@ void cpufreq_stats_record_transition(struct cpufreq_policy *policy, new_index = freq_table_get_index(stats, new_freq); /* We can't do stats->time_in_state[-1]= .. */ - if (old_index == -1 || new_index == -1 || old_index == new_index) + if (unlikely(old_index == -1 || new_index == -1 || old_index == new_index)) return; cpufreq_stats_update(stats, stats->last_time); From 96f60cddf7a1c5374296dd14355b519ee3245b3e Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 5 Oct 2020 13:26:04 +0530 Subject: [PATCH 228/265] cpufreq: stats: Enable stats for fast-switch as well Now that all the blockers are gone for enabling stats in fast-switching case, enable it. Signed-off-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq.c | 4 ++++ drivers/cpufreq/cpufreq_stats.c | 6 ------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 2ea245a6c0c0..2d0e2e464b14 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2072,8 +2072,12 @@ unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, target_freq = clamp_val(target_freq, policy->min, policy->max); freq = cpufreq_driver->fast_switch(policy, target_freq); + if (!freq) + return 0; + arch_set_freq_scale(policy->related_cpus, freq, policy->cpuinfo.max_freq); + cpufreq_stats_record_transition(policy, freq); return freq; } diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index bba04da3a278..8e7d64f34041 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -69,9 +69,6 @@ static ssize_t show_time_in_state(struct cpufreq_policy *policy, char *buf) ssize_t len = 0; int i; - if (policy->fast_switch_enabled) - return 0; - for (i = 0; i < stats->state_num; i++) { if (pending) { if (i == stats->last_index) @@ -115,9 +112,6 @@ static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf) ssize_t len = 0; int i, j, count; - if (policy->fast_switch_enabled) - return 0; - len += scnprintf(buf + len, PAGE_SIZE - len, " From : To\n"); len += scnprintf(buf + len, PAGE_SIZE - len, " : "); for (i = 0; i < stats->state_num; i++) { From 08d8c65e849d7579bafe2b03eab844d7860e3682 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 5 Oct 2020 13:26:05 +0530 Subject: [PATCH 229/265] cpufreq: Move traces and update to policy->cur to cpufreq core The cpufreq core handles the updates to policy->cur and recording of cpufreq trace events for all the governors except schedutil's fast switch case. Move that as well to cpufreq core for consistency and readability. Signed-off-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq.c | 7 +++++++ kernel/sched/cpufreq_schedutil.c | 12 +----------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 2d0e2e464b14..db00693a586a 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2068,6 +2068,7 @@ unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq) { unsigned int freq; + int cpu; target_freq = clamp_val(target_freq, policy->min, policy->max); freq = cpufreq_driver->fast_switch(policy, target_freq); @@ -2075,10 +2076,16 @@ unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, if (!freq) return 0; + policy->cur = freq; arch_set_freq_scale(policy->related_cpus, freq, policy->cpuinfo.max_freq); cpufreq_stats_record_transition(policy, freq); + if (trace_cpu_frequency_enabled()) { + for_each_cpu(cpu, policy->cpus) + trace_cpu_frequency(freq, cpu); + } + return freq; } EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index e39008242cf4..28f6d1ad608b 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -115,21 +115,11 @@ static void sugov_fast_switch(struct sugov_policy *sg_policy, u64 time, unsigned int next_freq) { struct cpufreq_policy *policy = sg_policy->policy; - int cpu; if (!sugov_update_next_freq(sg_policy, time, next_freq)) return; - next_freq = cpufreq_driver_fast_switch(policy, next_freq); - if (!next_freq) - return; - - policy->cur = next_freq; - - if (trace_cpu_frequency_enabled()) { - for_each_cpu(cpu, policy->cpus) - trace_cpu_frequency(next_freq, cpu); - } + cpufreq_driver_fast_switch(policy, next_freq); } static void sugov_deferred_update(struct sugov_policy *sg_policy, u64 time, From 428805c0c5e76ef643b1fbc893edfb636b3d8aef Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 25 Sep 2020 18:14:47 +0200 Subject: [PATCH 230/265] PM: hibernate: remove the bogus call to get_gendisk() in software_resume() get_gendisk grabs a reference on the disk and file operation, so this code will leak both of them while having absolutely no use for the gendisk itself. This effectively reverts commit 2df83fa4bce421f ("PM / Hibernate: Use get_gendisk to verify partition if resume_file is integer format") Signed-off-by: Christoph Hellwig Signed-off-by: Rafael J. Wysocki --- kernel/power/hibernate.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index e7aa57fb2fdc..7d0b99d2e696 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -948,17 +948,6 @@ static int software_resume(void) /* Check if the device is there */ swsusp_resume_device = name_to_dev_t(resume_file); - - /* - * name_to_dev_t is ineffective to verify parition if resume_file is in - * integer format. (e.g. major:minor) - */ - if (isdigit(resume_file[0]) && resume_wait) { - int partno; - while (!get_gendisk(swsusp_resume_device, &partno)) - msleep(10); - } - if (!swsusp_resume_device) { /* * Some device discovery might still be in progress; we need From c6e331312ebfb52b7186e5d82d517d68b4d2f2d8 Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Fri, 2 Oct 2020 07:10:12 +0200 Subject: [PATCH 231/265] PCI/ACPI: Whitelist hotplug ports for D3 if power managed by ACPI Recent laptops with dual AMD GPUs fail to suspend the discrete GPU, thus causing lockups on system sleep and high power consumption at runtime. The discrete GPU would normally be suspended to D3cold by turning off ACPI _PR3 Power Resources of the Root Port above the GPU. However on affected systems, the Root Port is hotplug-capable and pci_bridge_d3_possible() only allows hotplug ports to go to D3 if they belong to a Thunderbolt device or if the Root Port possesses a "HotPlugSupportInD3" ACPI property. Neither is the case on affected laptops. The reason for whitelisting only specific, known to work hotplug ports for D3 is that there have been reports of SkyLake Xeon-SP systems raising Hardware Error NMIs upon suspending their hotplug ports: https://lore.kernel.org/linux-pci/20170503180426.GA4058@otc-nc-03/ But if a hotplug port is power manageable by ACPI (as can be detected through presence of Power Resources and corresponding _PS0 and _PS3 methods) then it ought to be safe to suspend it to D3. To this end, amend acpi_pci_bridge_d3() to whitelist such ports for D3. Link: https://gitlab.freedesktop.org/drm/amd/-/issues/1222 Link: https://gitlab.freedesktop.org/drm/amd/-/issues/1252 Link: https://gitlab.freedesktop.org/drm/amd/-/issues/1304 Reported-and-tested-by: Arthur Borsboom Reported-and-tested-by: matoro Reported-by: Aaron Zakhrov Reported-by: Michal Rostecki Reported-by: Shai Coleman Signed-off-by: Lukas Wunner Acked-by: Alex Deucher Cc: 5.4+ # 5.4+ Signed-off-by: Rafael J. Wysocki --- drivers/pci/pci-acpi.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c index d5869a03f748..d9aa551f8423 100644 --- a/drivers/pci/pci-acpi.c +++ b/drivers/pci/pci-acpi.c @@ -944,6 +944,16 @@ static bool acpi_pci_bridge_d3(struct pci_dev *dev) if (!dev->is_hotplug_bridge) return false; + /* Assume D3 support if the bridge is power-manageable by ACPI. */ + adev = ACPI_COMPANION(&dev->dev); + if (!adev && !pci_dev_is_added(dev)) { + adev = acpi_pci_find_companion(&dev->dev); + ACPI_COMPANION_SET(&dev->dev, adev); + } + + if (adev && acpi_device_power_manageable(adev)) + return true; + /* * Look for a special _DSD property for the root port and if it * is set we know the hierarchy behind it supports D3 just fine. From 5e92442bb4121562231e6daf8a2d1306cb5f8805 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 5 Oct 2020 19:13:15 +0200 Subject: [PATCH 232/265] ACPI: EC: PM: Flush EC work unconditionally after wakeup Commit 607b9df63057 ("ACPI: EC: PM: Avoid flushing EC work when EC GPE is inactive") has been reported to cause some power button wakeup events to be missed on some systems, so modify acpi_ec_dispatch_gpe() to call acpi_ec_flush_work() unconditionally to effectively reverse the changes made by that commit. Also note that the problem which prompted commit 607b9df63057 is not reproducible any more on the affected machine. Fixes: 607b9df63057 ("ACPI: EC: PM: Avoid flushing EC work when EC GPE is inactive") Reported-by: Raymond Tan Cc: 5.4+ # 5.4+ Signed-off-by: Rafael J. Wysocki --- drivers/acpi/ec.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index fcddda3d6712..548cdbf7cca1 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -2019,12 +2019,11 @@ bool acpi_ec_dispatch_gpe(void) * to allow the caller to process events properly after that. */ ret = acpi_dispatch_gpe(NULL, first_ec->gpe); - if (ret == ACPI_INTERRUPT_HANDLED) { + if (ret == ACPI_INTERRUPT_HANDLED) pm_pr_dbg("ACPI EC GPE dispatched\n"); - /* Flush the event and query workqueues. */ - acpi_ec_flush_work(); - } + /* Flush the event and query workqueues. */ + acpi_ec_flush_work(); return false; } From e0e9ce390d7bc6a705653d4a8aa4ea92c9a65e53 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 5 Oct 2020 19:13:46 +0200 Subject: [PATCH 233/265] ACPI: EC: PM: Drop ec_no_wakeup check from acpi_ec_dispatch_gpe() It turns out that in some cases there are EC events to flush in acpi_ec_dispatch_gpe() even though the ec_no_wakeup kernel parameter is set and the EC GPE is disabled while sleeping, so drop the ec_no_wakeup check that prevents those events from being processed from acpi_ec_dispatch_gpe(). Reported-by: Todd Brandt Cc: 5.4+ # 5.4+ Signed-off-by: Rafael J. Wysocki --- drivers/acpi/ec.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index 548cdbf7cca1..e0cb1bcfffb2 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -2011,9 +2011,6 @@ bool acpi_ec_dispatch_gpe(void) if (acpi_any_gpe_status_set(first_ec->gpe)) return true; - if (ec_no_wakeup) - return false; - /* * Dispatch the EC GPE in-band, but do not report wakeup in any case * to allow the caller to process events properly after that. From 2486baae2cf6df73554144d0a4e40ae8809b54d4 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Mon, 5 Oct 2020 17:50:28 +0200 Subject: [PATCH 234/265] objtool: Allow nested externs to enable BUILD_BUG() Currently BUILD_BUG() macro is expanded to smth like the following: do { extern void __compiletime_assert_0(void) __attribute__((error("BUILD_BUG failed"))); if (!(!(1))) __compiletime_assert_0(); } while (0); If used in a function body this obviously would produce build errors with -Wnested-externs and -Werror. Build objtool with -Wno-nested-externs to enable BUILD_BUG() usage. Signed-off-by: Vasily Gorbik Signed-off-by: Josh Poimboeuf --- tools/objtool/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile index 33d1e3ca8efd..4ea9a833dde7 100644 --- a/tools/objtool/Makefile +++ b/tools/objtool/Makefile @@ -37,7 +37,7 @@ INCLUDES := -I$(srctree)/tools/include \ -I$(srctree)/tools/arch/$(HOSTARCH)/include/uapi \ -I$(srctree)/tools/arch/$(SRCARCH)/include \ -I$(srctree)/tools/objtool/arch/$(SRCARCH)/include -WARNINGS := $(EXTRA_WARNINGS) -Wno-switch-default -Wno-switch-enum -Wno-packed +WARNINGS := $(EXTRA_WARNINGS) -Wno-switch-default -Wno-switch-enum -Wno-packed -Wno-nested-externs CFLAGS := -Werror $(WARNINGS) $(KBUILD_HOSTCFLAGS) -g $(INCLUDES) $(LIBELF_FLAGS) LDFLAGS += $(LIBELF_LIBS) $(LIBSUBCMD) $(KBUILD_HOSTLDFLAGS) From 81c93798ef3ebd2109dc24371db3cc14cdf77777 Mon Sep 17 00:00:00 2001 From: Elia Devito Date: Sun, 4 Oct 2020 23:13:05 +0200 Subject: [PATCH 235/265] platform/x86: hp-wmi: add support for thermal policy HP Spectre notebooks (and probably other model as well) support up to 4 thermal policy: - HP Recommended - Performance - Cool - Quiet at least on HP Spectre x360 Convertible 15-df0xxx the firmware sets the thermal policy to default but hardcode the odvp0 variable to 1, this causes thermald to choose the wrong DPTF profile witch result in low performance when notebook is on AC, calling thermal policy write command allow firmware to correctly set the odvp0 variable. Signed-off-by: Elia Devito Link: https://lore.kernel.org/r/20201004211305.11628-1-eliadevito@gmail.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/hp-wmi.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/platform/x86/hp-wmi.c b/drivers/platform/x86/hp-wmi.c index 1762f335bac9..ecd477964d11 100644 --- a/drivers/platform/x86/hp-wmi.c +++ b/drivers/platform/x86/hp-wmi.c @@ -81,6 +81,7 @@ enum hp_wmi_commandtype { HPWMI_FEATURE2_QUERY = 0x0d, HPWMI_WIRELESS2_QUERY = 0x1b, HPWMI_POSTCODEERROR_QUERY = 0x2a, + HPWMI_THERMAL_POLICY_QUERY = 0x4c, }; enum hp_wmi_command { @@ -861,6 +862,26 @@ fail: return err; } +static int thermal_policy_setup(struct platform_device *device) +{ + int err, tp; + + tp = hp_wmi_read_int(HPWMI_THERMAL_POLICY_QUERY); + if (tp < 0) + return tp; + + /* + * call thermal policy write command to ensure that the firmware correctly + * sets the OEM variables for the DPTF + */ + err = hp_wmi_perform_query(HPWMI_THERMAL_POLICY_QUERY, HPWMI_WRITE, &tp, + sizeof(tp), 0); + if (err) + return err; + + return 0; +} + static int __init hp_wmi_bios_setup(struct platform_device *device) { /* clear detected rfkill devices */ @@ -872,6 +893,8 @@ static int __init hp_wmi_bios_setup(struct platform_device *device) if (hp_wmi_rfkill_setup(device)) hp_wmi_rfkill2_setup(device); + thermal_policy_setup(device); + return 0; } From 86836bac55f971995499978df4e62115d7baf5ef Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 6 Oct 2020 14:01:31 +0200 Subject: [PATCH 236/265] cpufreq: schedutil: Simplify sugov_fast_switch() Drop a redundant local variable definition from sugov_fast_switch() and rearrange the code in there to avoid the redundant logical negation. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar --- kernel/sched/cpufreq_schedutil.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 28f6d1ad608b..5ae7b4e6e8d6 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -114,12 +114,8 @@ static bool sugov_update_next_freq(struct sugov_policy *sg_policy, u64 time, static void sugov_fast_switch(struct sugov_policy *sg_policy, u64 time, unsigned int next_freq) { - struct cpufreq_policy *policy = sg_policy->policy; - - if (!sugov_update_next_freq(sg_policy, time, next_freq)) - return; - - cpufreq_driver_fast_switch(policy, next_freq); + if (sugov_update_next_freq(sg_policy, time, next_freq)) + cpufreq_driver_fast_switch(sg_policy->policy, next_freq); } static void sugov_deferred_update(struct sugov_policy *sg_policy, u64 time, From efad4240da949fc3249015065b95d708b72ae670 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 6 Oct 2020 21:43:43 +0200 Subject: [PATCH 237/265] cpufreq: stats: Add memory barrier to store_reset() There is nothing to prevent the CPU or the compiler from reordering the writes to stats->reset_time and stats->reset_pending in store_reset(), in which case the readers of stats->reset_time may see a stale value. Moreover, on 32-bit arches the write to reset_time cannot be completed in one go, so the readers of it may see a partially updated value in that case. To prevent that from happening, add a write memory barrier between the writes to stats->reset_time and stats->reset_pending in store_reset() and corresponding read memory barrier in the readers of stats->reset_time. Fixes: 40c3bd4cfa6f ("cpufreq: stats: Defer stats update to cpufreq_stats_record_transition()") Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar --- drivers/cpufreq/cpufreq_stats.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index 8e7d64f34041..1b1389face85 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -47,6 +47,11 @@ static void cpufreq_stats_reset_table(struct cpufreq_stats *stats) /* Adjust for the time elapsed since reset was requested */ WRITE_ONCE(stats->reset_pending, 0); + /* + * Prevent the reset_time read from being reordered before the + * reset_pending accesses in cpufreq_stats_record_transition(). + */ + smp_rmb(); cpufreq_stats_update(stats, READ_ONCE(stats->reset_time)); } @@ -71,10 +76,16 @@ static ssize_t show_time_in_state(struct cpufreq_policy *policy, char *buf) for (i = 0; i < stats->state_num; i++) { if (pending) { - if (i == stats->last_index) + if (i == stats->last_index) { + /* + * Prevent the reset_time read from occurring + * before the reset_pending read above. + */ + smp_rmb(); time = get_jiffies_64() - READ_ONCE(stats->reset_time); - else + } else { time = 0; + } } else { time = stats->time_in_state[i]; if (i == stats->last_index) @@ -99,6 +110,11 @@ static ssize_t store_reset(struct cpufreq_policy *policy, const char *buf, * avoid races. */ WRITE_ONCE(stats->reset_time, get_jiffies_64()); + /* + * The memory barrier below is to prevent the readers of reset_time from + * seeing a stale or partially updated value. + */ + smp_wmb(); WRITE_ONCE(stats->reset_pending, 1); return count; From 7566616fb968dcc9f11d3f6b57132d33acab4e35 Mon Sep 17 00:00:00 2001 From: Jonathan Doman Date: Wed, 30 Sep 2020 15:26:36 -0700 Subject: [PATCH 238/265] tools/power/x86/intel-speed-select: Fix missing base-freq core IDs The reported base-freq high-priority-cpu-list was potentially omitting some cpus, due to incorrectly using a logical core count to constrain the size of a physical punit core ID mask. We may need to read both high and low PBF CORE_MASK values regardless of the logical core count. Signed-off-by: Jonathan Doman Signed-off-by: Srinivas Pandruvada Signed-off-by: Hans de Goede --- .../x86/intel-speed-select/isst-config.c | 21 +++++++++++-------- .../power/x86/intel-speed-select/isst-core.c | 8 +++---- tools/power/x86/intel-speed-select/isst.h | 2 +- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/tools/power/x86/intel-speed-select/isst-config.c b/tools/power/x86/intel-speed-select/isst-config.c index 9f4b190f1d74..0bba5be8d1fd 100644 --- a/tools/power/x86/intel-speed-select/isst-config.c +++ b/tools/power/x86/intel-speed-select/isst-config.c @@ -545,20 +545,23 @@ static void set_cpu_present_cpu_mask(void) } } -int get_core_count(int pkg_id, int die_id) +int get_max_punit_core_id(int pkg_id, int die_id) { - int cnt = 0; + int max_id = 0; + int i; - if (pkg_id < MAX_PACKAGE_COUNT && die_id < MAX_DIE_PER_PACKAGE) { - int i; + for (i = 0; i < topo_max_cpus; ++i) + { + if (!CPU_ISSET_S(i, present_cpumask_size, present_cpumask)) + continue; - for (i = 0; i < sizeof(long long) * 8; ++i) { - if (core_mask[pkg_id][die_id] & (1ULL << i)) - cnt++; - } + if (cpu_map[i].pkg_id == pkg_id && + cpu_map[i].die_id == die_id && + cpu_map[i].punit_cpu_core > max_id) + max_id = cpu_map[i].punit_cpu_core; } - return cnt; + return max_id; } int get_cpu_count(int pkg_id, int die_id) diff --git a/tools/power/x86/intel-speed-select/isst-core.c b/tools/power/x86/intel-speed-select/isst-core.c index a7f4337c5777..1d7ecb54352e 100644 --- a/tools/power/x86/intel-speed-select/isst-core.c +++ b/tools/power/x86/intel-speed-select/isst-core.c @@ -396,7 +396,7 @@ int isst_get_pbf_info(int cpu, int level, struct isst_pbf_info *pbf_info) { struct isst_pkg_ctdp_level_info ctdp_level; struct isst_pkg_ctdp pkg_dev; - int i, ret, core_cnt, max; + int i, ret, max_punit_core, max_mask_index; unsigned int req, resp; ret = isst_get_ctdp_levels(cpu, &pkg_dev); @@ -421,10 +421,10 @@ int isst_get_pbf_info(int cpu, int level, struct isst_pbf_info *pbf_info) pbf_info->core_cpumask_size = alloc_cpu_set(&pbf_info->core_cpumask); - core_cnt = get_core_count(get_physical_package_id(cpu), get_physical_die_id(cpu)); - max = core_cnt > 32 ? 2 : 1; + max_punit_core = get_max_punit_core_id(get_physical_package_id(cpu), get_physical_die_id(cpu)); + max_mask_index = max_punit_core > 32 ? 2 : 1; - for (i = 0; i < max; ++i) { + for (i = 0; i < max_mask_index; ++i) { unsigned long long mask; int count; diff --git a/tools/power/x86/intel-speed-select/isst.h b/tools/power/x86/intel-speed-select/isst.h index 094ba4589a9c..29715e9c2e06 100644 --- a/tools/power/x86/intel-speed-select/isst.h +++ b/tools/power/x86/intel-speed-select/isst.h @@ -170,7 +170,7 @@ struct isst_pkg_ctdp { extern int get_topo_max_cpus(void); extern int get_cpu_count(int pkg_id, int die_id); -extern int get_core_count(int pkg_id, int die_id); +extern int get_max_punit_core_id(int pkg_id, int die_id); /* Common interfaces */ FILE *get_output_file(void); From e529412f3211071b4787a8e7e153c8ced3c22a28 Mon Sep 17 00:00:00 2001 From: Srinivas Pandruvada Date: Tue, 6 Oct 2020 06:26:31 -0700 Subject: [PATCH 239/265] tools/power/x86/intel-speed-select: Update version for v5.10 Update version for changes released with v5.10 kernel release. Signed-off-by: Srinivas Pandruvada Signed-off-by: Hans de Goede --- tools/power/x86/intel-speed-select/isst-config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/power/x86/intel-speed-select/isst-config.c b/tools/power/x86/intel-speed-select/isst-config.c index 0bba5be8d1fd..cd089a505859 100644 --- a/tools/power/x86/intel-speed-select/isst-config.c +++ b/tools/power/x86/intel-speed-select/isst-config.c @@ -15,7 +15,7 @@ struct process_cmd_struct { int arg; }; -static const char *version_str = "v1.5"; +static const char *version_str = "v1.6"; static const int supported_api_ver = 1; static struct isst_if_platform_info isst_platform_info; static char *progname; From e973f1d372dca5ac1d107ec6134f72e566fdf968 Mon Sep 17 00:00:00 2001 From: "David E. Box" Date: Tue, 6 Oct 2020 15:47:00 -0700 Subject: [PATCH 240/265] platform/x86: pmc_core: Use descriptive names for LPM registers TigerLake Lower Power Mode (LPM) registers are grouped by functionality but were given simple enumerated names in the code (lpm0, lpm1, ...). Instead, give the register blocks names that describe their usage. Suggested-by: Andy Shevchenko Signed-off-by: David E. Box Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20201006224702.12697-2-david.e.box@linux.intel.com Signed-off-by: Hans de Goede --- drivers/platform/x86/intel_pmc_core.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/platform/x86/intel_pmc_core.c b/drivers/platform/x86/intel_pmc_core.c index 338ea5222555..ed9fdf7c8928 100644 --- a/drivers/platform/x86/intel_pmc_core.c +++ b/drivers/platform/x86/intel_pmc_core.c @@ -409,7 +409,7 @@ static const struct pmc_reg_map icl_reg_map = { .ltr_ignore_max = ICL_NUM_IP_IGN_ALLOWED, }; -static const struct pmc_bit_map tgl_lpm0_map[] = { +static const struct pmc_bit_map tgl_clocksource_status_map[] = { {"USB2PLL_OFF_STS", BIT(18)}, {"PCIe/USB3.1_Gen2PLL_OFF_STS", BIT(19)}, {"PCIe_Gen3PLL_OFF_STS", BIT(20)}, @@ -425,7 +425,7 @@ static const struct pmc_bit_map tgl_lpm0_map[] = { {} }; -static const struct pmc_bit_map tgl_lpm1_map[] = { +static const struct pmc_bit_map tgl_power_gating_status_map[] = { {"SPI_PG_STS", BIT(2)}, {"xHCI_PG_STS", BIT(3)}, {"PCIe_Ctrller_A_PG_STS", BIT(4)}, @@ -453,7 +453,7 @@ static const struct pmc_bit_map tgl_lpm1_map[] = { {} }; -static const struct pmc_bit_map tgl_lpm2_map[] = { +static const struct pmc_bit_map tgl_d3_status_map[] = { {"ADSP_D3_STS", BIT(0)}, {"SATA_D3_STS", BIT(1)}, {"xHCI0_D3_STS", BIT(2)}, @@ -468,7 +468,7 @@ static const struct pmc_bit_map tgl_lpm2_map[] = { {} }; -static const struct pmc_bit_map tgl_lpm3_map[] = { +static const struct pmc_bit_map tgl_vnn_req_status_map[] = { {"GPIO_COM0_VNN_REQ_STS", BIT(1)}, {"GPIO_COM1_VNN_REQ_STS", BIT(2)}, {"GPIO_COM2_VNN_REQ_STS", BIT(3)}, @@ -493,7 +493,7 @@ static const struct pmc_bit_map tgl_lpm3_map[] = { {} }; -static const struct pmc_bit_map tgl_lpm4_map[] = { +static const struct pmc_bit_map tgl_vnn_misc_status_map[] = { {"CPU_C10_REQ_STS_0", BIT(0)}, {"PCIe_LPM_En_REQ_STS_3", BIT(3)}, {"ITH_REQ_STS_5", BIT(5)}, @@ -509,7 +509,7 @@ static const struct pmc_bit_map tgl_lpm4_map[] = { {} }; -static const struct pmc_bit_map tgl_lpm5_map[] = { +static const struct pmc_bit_map tgl_signal_status_map[] = { {"LSX_Wake0_En_STS", BIT(0)}, {"LSX_Wake0_Pol_STS", BIT(1)}, {"LSX_Wake1_En_STS", BIT(2)}, @@ -546,12 +546,12 @@ static const struct pmc_bit_map tgl_lpm5_map[] = { }; static const struct pmc_bit_map *tgl_lpm_maps[] = { - tgl_lpm0_map, - tgl_lpm1_map, - tgl_lpm2_map, - tgl_lpm3_map, - tgl_lpm4_map, - tgl_lpm5_map, + tgl_clocksource_status_map, + tgl_power_gating_status_map, + tgl_d3_status_map, + tgl_vnn_req_status_map, + tgl_vnn_misc_status_map, + tgl_signal_status_map, NULL }; From 652036bd5be0ac94fd1db851d72ece8ee133c74d Mon Sep 17 00:00:00 2001 From: Gayatri Kammela Date: Tue, 6 Oct 2020 15:47:01 -0700 Subject: [PATCH 241/265] platform/x86: intel_pmc_core: Fix TigerLake power gating status map TigerLake's LPM power gating status register has errors in the bit-to-name mapping as well as with the marked reserved bits according to the actual implementation. Hence, update the right bit-to-name mapping and the reserved bits in accordance with actual implementation. Cc: Srinivas Pandruvada Cc: Andy Shevchenko Cc: David E. Box Signed-off-by: Gayatri Kammela Signed-off-by: David E. Box Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20201006224702.12697-3-david.e.box@linux.intel.com Signed-off-by: Hans de Goede --- drivers/platform/x86/intel_pmc_core.c | 48 +++++++++++++-------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/drivers/platform/x86/intel_pmc_core.c b/drivers/platform/x86/intel_pmc_core.c index ed9fdf7c8928..cf4006e08c69 100644 --- a/drivers/platform/x86/intel_pmc_core.c +++ b/drivers/platform/x86/intel_pmc_core.c @@ -426,30 +426,30 @@ static const struct pmc_bit_map tgl_clocksource_status_map[] = { }; static const struct pmc_bit_map tgl_power_gating_status_map[] = { - {"SPI_PG_STS", BIT(2)}, - {"xHCI_PG_STS", BIT(3)}, - {"PCIe_Ctrller_A_PG_STS", BIT(4)}, - {"PCIe_Ctrller_B_PG_STS", BIT(5)}, - {"PCIe_Ctrller_C_PG_STS", BIT(6)}, - {"GBE_PG_STS", BIT(7)}, - {"SATA_PG_STS", BIT(8)}, - {"HDA0_PG_STS", BIT(9)}, - {"HDA1_PG_STS", BIT(10)}, - {"HDA2_PG_STS", BIT(11)}, - {"HDA3_PG_STS", BIT(12)}, - {"PCIe_Ctrller_D_PG_STS", BIT(13)}, - {"ISIO_PG_STS", BIT(14)}, - {"SMB_PG_STS", BIT(16)}, - {"ISH_PG_STS", BIT(17)}, - {"ITH_PG_STS", BIT(19)}, - {"SDX_PG_STS", BIT(20)}, - {"xDCI_PG_STS", BIT(25)}, - {"DCI_PG_STS", BIT(26)}, - {"CSME0_PG_STS", BIT(27)}, - {"CSME_KVM_PG_STS", BIT(28)}, - {"CSME1_PG_STS", BIT(29)}, - {"CSME_CLINK_PG_STS", BIT(30)}, - {"CSME2_PG_STS", BIT(31)}, + {"CSME_PG_STS", BIT(0)}, + {"SATA_PG_STS", BIT(1)}, + {"xHCI_PG_STS", BIT(2)}, + {"UFSX2_PG_STS", BIT(3)}, + {"OTG_PG_STS", BIT(5)}, + {"SPA_PG_STS", BIT(6)}, + {"SPB_PG_STS", BIT(7)}, + {"SPC_PG_STS", BIT(8)}, + {"SPD_PG_STS", BIT(9)}, + {"SPE_PG_STS", BIT(10)}, + {"SPF_PG_STS", BIT(11)}, + {"LSX_PG_STS", BIT(13)}, + {"P2SB_PG_STS", BIT(14)}, + {"PSF_PG_STS", BIT(15)}, + {"SBR_PG_STS", BIT(16)}, + {"OPIDMI_PG_STS", BIT(17)}, + {"THC0_PG_STS", BIT(18)}, + {"THC1_PG_STS", BIT(19)}, + {"GBETSN_PG_STS", BIT(20)}, + {"GBE_PG_STS", BIT(21)}, + {"LPSS_PG_STS", BIT(22)}, + {"MMP_UFSX2_PG_STS", BIT(23)}, + {"MMP_UFSX2B_PG_STS", BIT(24)}, + {"FIA_PG_STS", BIT(25)}, {} }; From 025f26de7fa0fc40c8baf6c19fb273500f3321f0 Mon Sep 17 00:00:00 2001 From: Gayatri Kammela Date: Tue, 6 Oct 2020 15:47:02 -0700 Subject: [PATCH 242/265] platform/x86: intel_pmc_core: Fix the slp_s0 counter displayed value slp_s0 counter value displayed via debugfs interface is calculated by multiplying the granularity for crystal oscillator tick as 100us with the value read from using slp_s0 offset. But the granularity of the tick varies from platform to platform and it needs to be fixed. Hence, specify granularity of the tick for each platform, so that the value of the slp_s0 counter is accurate. Signed-off-by: Gayatri Kammela Signed-off-by: David E. Box Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20201006224702.12697-4-david.e.box@linux.intel.com Signed-off-by: Hans de Goede --- drivers/platform/x86/intel_pmc_core.c | 10 +++++++--- drivers/platform/x86/intel_pmc_core.h | 5 ++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/platform/x86/intel_pmc_core.c b/drivers/platform/x86/intel_pmc_core.c index cf4006e08c69..122eb53eb595 100644 --- a/drivers/platform/x86/intel_pmc_core.c +++ b/drivers/platform/x86/intel_pmc_core.c @@ -154,6 +154,7 @@ static const struct pmc_reg_map spt_reg_map = { .ltr_show_sts = spt_ltr_show_map, .msr_sts = msr_map, .slp_s0_offset = SPT_PMC_SLP_S0_RES_COUNTER_OFFSET, + .slp_s0_res_counter_step = SPT_PMC_SLP_S0_RES_COUNTER_STEP, .ltr_ignore_offset = SPT_PMC_LTR_IGNORE_OFFSET, .regmap_length = SPT_PMC_MMIO_REG_LEN, .ppfear0_offset = SPT_PMC_XRAM_PPFEAR0A, @@ -380,6 +381,7 @@ static const struct pmc_bit_map cnp_ltr_show_map[] = { static const struct pmc_reg_map cnp_reg_map = { .pfear_sts = ext_cnp_pfear_map, .slp_s0_offset = CNP_PMC_SLP_S0_RES_COUNTER_OFFSET, + .slp_s0_res_counter_step = SPT_PMC_SLP_S0_RES_COUNTER_STEP, .slps0_dbg_maps = cnp_slps0_dbg_maps, .ltr_show_sts = cnp_ltr_show_map, .msr_sts = msr_map, @@ -396,6 +398,7 @@ static const struct pmc_reg_map cnp_reg_map = { static const struct pmc_reg_map icl_reg_map = { .pfear_sts = ext_icl_pfear_map, .slp_s0_offset = CNP_PMC_SLP_S0_RES_COUNTER_OFFSET, + .slp_s0_res_counter_step = ICL_PMC_SLP_S0_RES_COUNTER_STEP, .slps0_dbg_maps = cnp_slps0_dbg_maps, .ltr_show_sts = cnp_ltr_show_map, .msr_sts = msr_map, @@ -558,6 +561,7 @@ static const struct pmc_bit_map *tgl_lpm_maps[] = { static const struct pmc_reg_map tgl_reg_map = { .pfear_sts = ext_tgl_pfear_map, .slp_s0_offset = CNP_PMC_SLP_S0_RES_COUNTER_OFFSET, + .slp_s0_res_counter_step = TGL_PMC_SLP_S0_RES_COUNTER_STEP, .ltr_show_sts = cnp_ltr_show_map, .msr_sts = msr_map, .ltr_ignore_offset = CNP_PMC_LTR_IGNORE_OFFSET, @@ -586,9 +590,9 @@ static inline void pmc_core_reg_write(struct pmc_dev *pmcdev, int reg_offset, writel(val, pmcdev->regbase + reg_offset); } -static inline u64 pmc_core_adjust_slp_s0_step(u32 value) +static inline u64 pmc_core_adjust_slp_s0_step(struct pmc_dev *pmcdev, u32 value) { - return (u64)value * SPT_PMC_SLP_S0_RES_COUNTER_STEP; + return (u64)value * pmcdev->map->slp_s0_res_counter_step; } static int pmc_core_dev_state_get(void *data, u64 *val) @@ -598,7 +602,7 @@ static int pmc_core_dev_state_get(void *data, u64 *val) u32 value; value = pmc_core_reg_read(pmcdev, map->slp_s0_offset); - *val = pmc_core_adjust_slp_s0_step(value); + *val = pmc_core_adjust_slp_s0_step(pmcdev, value); return 0; } diff --git a/drivers/platform/x86/intel_pmc_core.h b/drivers/platform/x86/intel_pmc_core.h index 5eae55d80226..f33cd2c34835 100644 --- a/drivers/platform/x86/intel_pmc_core.h +++ b/drivers/platform/x86/intel_pmc_core.h @@ -30,7 +30,7 @@ #define SPT_PMC_MPHY_CORE_STS_1 0x1142 #define SPT_PMC_MPHY_COM_STS_0 0x1155 #define SPT_PMC_MMIO_REG_LEN 0x1000 -#define SPT_PMC_SLP_S0_RES_COUNTER_STEP 0x64 +#define SPT_PMC_SLP_S0_RES_COUNTER_STEP 0x68 #define PMC_BASE_ADDR_MASK ~(SPT_PMC_MMIO_REG_LEN - 1) #define MTPMC_MASK 0xffff0000 #define PPFEAR_MAX_NUM_ENTRIES 12 @@ -185,8 +185,10 @@ enum ppfear_regs { #define ICL_PPFEAR_NUM_ENTRIES 9 #define ICL_NUM_IP_IGN_ALLOWED 20 #define ICL_PMC_LTR_WIGIG 0x1BFC +#define ICL_PMC_SLP_S0_RES_COUNTER_STEP 0x64 #define TGL_NUM_IP_IGN_ALLOWED 22 +#define TGL_PMC_SLP_S0_RES_COUNTER_STEP 0x7A /* * Tigerlake Power Management Controller register offsets @@ -245,6 +247,7 @@ struct pmc_reg_map { const struct pmc_bit_map *msr_sts; const struct pmc_bit_map **lpm_sts; const u32 slp_s0_offset; + const int slp_s0_res_counter_step; const u32 ltr_ignore_offset; const int regmap_length; const u32 ppfear0_offset; From 3976c6e365d7d50cbc96e92b0c537f5e8c744373 Mon Sep 17 00:00:00 2001 From: Gayatri Kammela Date: Tue, 6 Oct 2020 20:51:05 -0700 Subject: [PATCH 243/265] platform/x86: intel_pmc_core: Clean up: Remove the duplicate comments and reorganize Some of the Cannon Lake PCH IPs are reused by most of the platforms such as Ice Lake, Tiger Lake, Elkhart Lake, Jasper Lake and can be reused by future platforms as well. The same was mentioned via comments not once but twice in an array of bit map structs for Cannon Lake (cnp_pfear_map). Hence, remove the duplicate comments and reorganize them. Cc: Srinivas Pandruvada Cc: Andy Shevchenko Cc: David E. Box Cc: Tony Luck Cc: Rui Zhang Suggested-by: Dave Hansen Signed-off-by: Gayatri Kammela Signed-off-by: David E. Box Reviewed-by: Andy Shevchenko Reviewed-by: Rajneesh Bhardwaj Link: https://lore.kernel.org/r/20201007035108.31078-2-david.e.box@linux.intel.com Signed-off-by: Hans de Goede --- drivers/platform/x86/intel_pmc_core.c | 32 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/drivers/platform/x86/intel_pmc_core.c b/drivers/platform/x86/intel_pmc_core.c index 122eb53eb595..c6d71662d398 100644 --- a/drivers/platform/x86/intel_pmc_core.c +++ b/drivers/platform/x86/intel_pmc_core.c @@ -118,6 +118,10 @@ static const struct pmc_bit_map spt_pfear_map[] = { }; static const struct pmc_bit_map *ext_spt_pfear_map[] = { + /* + * Check intel_pmc_core_ids[] users of spt_reg_map for + * a list of core SoCs using this. + */ spt_pfear_map, NULL }; @@ -167,7 +171,6 @@ static const struct pmc_reg_map spt_reg_map = { /* Cannon Lake: PGD PFET Enable Ack Status Register(s) bitmap */ static const struct pmc_bit_map cnp_pfear_map[] = { - /* Reserved for Cannon Lake but valid for Comet Lake */ {"PMC", BIT(0)}, {"OPI-DMI", BIT(1)}, {"SPI/eSPI", BIT(2)}, @@ -193,10 +196,6 @@ static const struct pmc_bit_map cnp_pfear_map[] = { {"SDX", BIT(4)}, {"SPE", BIT(5)}, {"Fuse", BIT(6)}, - /* - * Reserved for Cannon Lake but valid for Ice Lake, Comet Lake, - * Tiger Lake, Elkhart Lake and Jasper Lake. - */ {"SBR8", BIT(7)}, {"CSME_FSC", BIT(0)}, @@ -240,10 +239,6 @@ static const struct pmc_bit_map cnp_pfear_map[] = { {"HDA_PGD4", BIT(2)}, {"HDA_PGD5", BIT(3)}, {"HDA_PGD6", BIT(4)}, - /* - * Reserved for Cannon Lake but valid for Ice Lake, Comet Lake, - * Tiger Lake, ELkhart Lake and Jasper Lake. - */ {"PSF6", BIT(5)}, {"PSF7", BIT(6)}, {"PSF8", BIT(7)}, @@ -251,12 +246,15 @@ static const struct pmc_bit_map cnp_pfear_map[] = { }; static const struct pmc_bit_map *ext_cnp_pfear_map[] = { + /* + * Check intel_pmc_core_ids[] users of cnp_reg_map for + * a list of core SoCs using this. + */ cnp_pfear_map, NULL }; static const struct pmc_bit_map icl_pfear_map[] = { - /* Ice Lake and Jasper Lake generation onwards only */ {"RES_65", BIT(0)}, {"RES_66", BIT(1)}, {"RES_67", BIT(2)}, @@ -269,13 +267,16 @@ static const struct pmc_bit_map icl_pfear_map[] = { }; static const struct pmc_bit_map *ext_icl_pfear_map[] = { + /* + * Check intel_pmc_core_ids[] users of icl_reg_map for + * a list of core SoCs using this. + */ cnp_pfear_map, icl_pfear_map, NULL }; static const struct pmc_bit_map tgl_pfear_map[] = { - /* Tiger Lake and Elkhart Lake generation onwards only */ {"PSF9", BIT(0)}, {"RES_66", BIT(1)}, {"RES_67", BIT(2)}, @@ -287,6 +288,10 @@ static const struct pmc_bit_map tgl_pfear_map[] = { }; static const struct pmc_bit_map *ext_tgl_pfear_map[] = { + /* + * Check intel_pmc_core_ids[] users of tgl_reg_map for + * a list of core SoCs using this. + */ cnp_pfear_map, tgl_pfear_map, NULL @@ -370,7 +375,10 @@ static const struct pmc_bit_map cnp_ltr_show_map[] = { {"ISH", CNP_PMC_LTR_ISH}, {"UFSX2", CNP_PMC_LTR_UFSX2}, {"EMMC", CNP_PMC_LTR_EMMC}, - /* Reserved for Cannon Lake but valid for Ice Lake */ + /* + * Check intel_pmc_core_ids[] users of cnp_reg_map for + * a list of core SoCs using this. + */ {"WIGIG", ICL_PMC_LTR_WIGIG}, /* Below two cannot be used for LTR_IGNORE */ {"CURRENT_PLATFORM", CNP_PMC_LTR_CUR_PLT}, From 68cb1a977e5e8b9531e7bf4fbfbde5ce1d19537b Mon Sep 17 00:00:00 2001 From: Gayatri Kammela Date: Tue, 6 Oct 2020 20:51:06 -0700 Subject: [PATCH 244/265] platform/x86: intel_pmc_core: Add Intel RocketLake (RKL) support Add RocketLake to the list of the platforms that intel_pmc_core driver supports for pmc_core device. RocketLake reuses all the TigerLake PCH IPs. Cc: Srinivas Pandruvada Cc: Andy Shevchenko Cc: David E. Box Cc: Tony Luck Cc: Rui Zhang Signed-off-by: Gayatri Kammela Signed-off-by: David E. Box Reviewed-by: Andy Shevchenko Reviewed-by: Rajneesh Bhardwaj Link: https://lore.kernel.org/r/20201007035108.31078-3-david.e.box@linux.intel.com Signed-off-by: Hans de Goede --- drivers/platform/x86/intel_pmc_core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/platform/x86/intel_pmc_core.c b/drivers/platform/x86/intel_pmc_core.c index c6d71662d398..da316000785e 100644 --- a/drivers/platform/x86/intel_pmc_core.c +++ b/drivers/platform/x86/intel_pmc_core.c @@ -1159,6 +1159,7 @@ static const struct x86_cpu_id intel_pmc_core_ids[] = { X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &tgl_reg_map), X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT, &tgl_reg_map), X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_L, &icl_reg_map), + X86_MATCH_INTEL_FAM6_MODEL(ROCKETLAKE, &tgl_reg_map), {} }; From 46461f8709ed1998bdfae35296ba63ab12ec50a9 Mon Sep 17 00:00:00 2001 From: Gayatri Kammela Date: Tue, 6 Oct 2020 20:51:07 -0700 Subject: [PATCH 245/265] platform/x86: intel_pmc_core: fix: Replace dev_dbg macro with dev_info() dev_dbg macro is used to dump the debug registers in resume from an S0ix failure. However, when CONFIG_DYNAMIC_DEBUG is not set, the user may not be able to find the debug dump on an S0ix failure which defeats the purpose. The output of these messages is already controlled by a module parameter, warn_on_s0ix_failures, making it a 2 step process to enable anyway when CONFIG_DYNAMIC_DEBUG is set. Hence, replace dev_dbg with dev_info, allowing the control of the messages solely through the module parameter which is N by default. Fixes commit 913f984a8347 ("platform/x86: intel_pmc_core: Add an additional parameter to pmc_core_lpm_display()") Cc: Srinivas Pandruvada Cc: Andy Shevchenko Cc: David E. Box Signed-off-by: Gayatri Kammela Signed-off-by: David E. Box Reviewed-by: Andy Shevchenko Reviewed-by: Rajneesh Bhardwaj Link: https://lore.kernel.org/r/20201007035108.31078-4-david.e.box@linux.intel.com Signed-off-by: Hans de Goede --- drivers/platform/x86/intel_pmc_core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/platform/x86/intel_pmc_core.c b/drivers/platform/x86/intel_pmc_core.c index da316000785e..3e5fe66333f1 100644 --- a/drivers/platform/x86/intel_pmc_core.c +++ b/drivers/platform/x86/intel_pmc_core.c @@ -640,7 +640,7 @@ static void pmc_core_slps0_display(struct pmc_dev *pmcdev, struct device *dev, offset += 4; while (map->name) { if (dev) - dev_dbg(dev, "SLP_S0_DBG: %-32s\tState: %s\n", + dev_info(dev, "SLP_S0_DBG: %-32s\tState: %s\n", map->name, data & map->bit_mask ? "Yes" : "No"); if (s) @@ -683,7 +683,7 @@ static void pmc_core_lpm_display(struct pmc_dev *pmcdev, struct device *dev, for (idx = 0; idx < arr_size; idx++) { if (dev) - dev_dbg(dev, "\nLPM_%s_%d:\t0x%x\n", str, idx, + dev_info(dev, "\nLPM_%s_%d:\t0x%x\n", str, idx, lpm_regs[idx]); if (s) seq_printf(s, "\nLPM_%s_%d:\t0x%x\n", str, idx, @@ -691,7 +691,7 @@ static void pmc_core_lpm_display(struct pmc_dev *pmcdev, struct device *dev, for (index = 0; maps[idx][index].name && index < len; index++) { bit_mask = maps[idx][index].bit_mask; if (dev) - dev_dbg(dev, "%-30s %-30d\n", + dev_info(dev, "%-30s %-30d\n", maps[idx][index].name, lpm_regs[idx] & bit_mask ? 1 : 0); if (s) From d0e21c24de71f0c46491d2ccf3d511447a74d2c7 Mon Sep 17 00:00:00 2001 From: Gayatri Kammela Date: Tue, 6 Oct 2020 20:51:08 -0700 Subject: [PATCH 246/265] MAINTAINERS: Update maintainers for pmc_core driver Update MAINTAINERS file for pmc_core driver to reflect the current maintainers. Cc: Vishwanath Somayaji Cc: Andy Shevchenko Cc: Srinivas Pandruvada Cc: David E. Box Acked-by: David E. Box Acked-by: Vishwanath Somayaji Signed-off-by: Gayatri Kammela Signed-off-by: David E. Box Reviewed-by: Andy Shevchenko Reviewed-by: Rajneesh Bhardwaj Link: https://lore.kernel.org/r/20201007035108.31078-5-david.e.box@linux.intel.com Signed-off-by: Hans de Goede --- MAINTAINERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index b1c97a16b6ce..edd2b19c240f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8905,8 +8905,8 @@ F: arch/x86/include/asm/intel_punit_ipc.h F: drivers/platform/x86/intel_punit_ipc.c INTEL PMC CORE DRIVER -M: Rajneesh Bhardwaj -M: Vishwanath Somayaji +M: Rajneesh Bhardwaj +M: David E Box L: platform-driver-x86@vger.kernel.org S: Maintained F: drivers/platform/x86/intel_pmc_core* From c071afcea6ecf24a3c119f25ce9f71ffd55b5dc2 Mon Sep 17 00:00:00 2001 From: Vadim Pasternak Date: Wed, 23 Sep 2020 20:20:49 +0300 Subject: [PATCH 247/265] platform/x86: mlx-platform: Remove PSU EEPROM configuration Remove PSU EEPROM configuration for systems class equipped with Mellanox chip Spectrume-2. Till now all the systems from this class used few types of power units, all equipped with EEPROM device with address space two bytes. Thus, all these devices have been handled by EEPROM driver "24c32". There is a new requirement is to support power unit replacement by "off the shelf" device, matching electrical required parameters. Such device could be equipped with different EEPROM type, which could be one byte address space addressing or even could be not equipped with EEPROM. In such case "24c32" will not work. Fixes: 1bd42d94ccab ("platform/x86: mlx-platform: Add support for new 200G IB and Ethernet systems") Signed-off-by: Vadim Pasternak Reviewed-by: Hans de Goede Link: https://lore.kernel.org/r/20200923172053.26296-2-vadimp@nvidia.com Signed-off-by: Hans de Goede --- drivers/platform/x86/mlx-platform.c | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/drivers/platform/x86/mlx-platform.c b/drivers/platform/x86/mlx-platform.c index 1506ec0a4777..04a745095c37 100644 --- a/drivers/platform/x86/mlx-platform.c +++ b/drivers/platform/x86/mlx-platform.c @@ -328,15 +328,6 @@ static struct i2c_board_info mlxplat_mlxcpld_psu[] = { }, }; -static struct i2c_board_info mlxplat_mlxcpld_ng_psu[] = { - { - I2C_BOARD_INFO("24c32", 0x51), - }, - { - I2C_BOARD_INFO("24c32", 0x50), - }, -}; - static struct i2c_board_info mlxplat_mlxcpld_pwr[] = { { I2C_BOARD_INFO("dps460", 0x59), @@ -770,15 +761,13 @@ static struct mlxreg_core_data mlxplat_mlxcpld_default_ng_psu_items_data[] = { .label = "psu1", .reg = MLXPLAT_CPLD_LPC_REG_PSU_OFFSET, .mask = BIT(0), - .hpdev.brdinfo = &mlxplat_mlxcpld_ng_psu[0], - .hpdev.nr = MLXPLAT_CPLD_PSU_MSNXXXX_NR, + .hpdev.nr = MLXPLAT_CPLD_NR_NONE, }, { .label = "psu2", .reg = MLXPLAT_CPLD_LPC_REG_PSU_OFFSET, .mask = BIT(1), - .hpdev.brdinfo = &mlxplat_mlxcpld_ng_psu[1], - .hpdev.nr = MLXPLAT_CPLD_PSU_MSNXXXX_NR, + .hpdev.nr = MLXPLAT_CPLD_NR_NONE, }, }; From 638eae9bc7eb1012d1e0f5a8fd4db46447f822e9 Mon Sep 17 00:00:00 2001 From: Vadim Pasternak Date: Wed, 23 Sep 2020 20:20:51 +0300 Subject: [PATCH 248/265] platform_data/mlxreg: Update module license Update license to SPDX-License. Signed-off-by: Vadim Pasternak Reviewed-by: Hans de Goede Link: https://lore.kernel.org/r/20200923172053.26296-4-vadimp@nvidia.com Signed-off-by: Hans de Goede --- include/linux/platform_data/mlxreg.h | 32 ++-------------------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/include/linux/platform_data/mlxreg.h b/include/linux/platform_data/mlxreg.h index 1af9c01563f9..0a727d405a7a 100644 --- a/include/linux/platform_data/mlxreg.h +++ b/include/linux/platform_data/mlxreg.h @@ -1,34 +1,6 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ /* - * Copyright (c) 2017 Mellanox Technologies. All rights reserved. - * Copyright (c) 2017 Vadim Pasternak - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the names of the copyright holders nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * Alternatively, this software may be distributed under the terms of the - * GNU General Public License ("GPL") version 2 as published by the Free - * Software Foundation. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * Copyright (C) 2017-2020 Mellanox Technologies Ltd. */ #ifndef __LINUX_PLATFORM_DATA_MLXREG_H From d2f3ab5b6b05f67817f3bd3f2fda5f0126e95a62 Mon Sep 17 00:00:00 2001 From: Vadim Pasternak Date: Wed, 23 Sep 2020 20:20:52 +0300 Subject: [PATCH 249/265] platform_data/mlxreg: Extend core platform structure Add 'capability' field to structure 'mlxreg_core_platform_data'. The purpose of this filed to indicate the actual number of the components within the particular group. Such components could be, for example the number of the FAN drawers. Some systems are equipped with FAN drawers with one tachometer inside, others with FAN drawers with several tachometers inside. Signed-off-by: Vadim Pasternak Reviewed-by: Hans de Goede Link: https://lore.kernel.org/r/20200923172053.26296-5-vadimp@nvidia.com Signed-off-by: Hans de Goede --- include/linux/platform_data/mlxreg.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/platform_data/mlxreg.h b/include/linux/platform_data/mlxreg.h index 0a727d405a7a..101333fe2b8d 100644 --- a/include/linux/platform_data/mlxreg.h +++ b/include/linux/platform_data/mlxreg.h @@ -109,6 +109,7 @@ struct mlxreg_core_item { * @features: supported features of device; * @version: implementation version; * @identity: device identity name; + * @capability: device capability register; */ struct mlxreg_core_platform_data { struct mlxreg_core_data *data; @@ -117,6 +118,7 @@ struct mlxreg_core_platform_data { u32 features; u32 version; char identity[MLXREG_CORE_LABEL_MAX_SIZE]; + u32 capability; }; /** From 47a514b642cf144d6067e219474f08cde562b840 Mon Sep 17 00:00:00 2001 From: Vadim Pasternak Date: Wed, 23 Sep 2020 20:20:53 +0300 Subject: [PATCH 250/265] platform/x86: mlx-platform: Add capability field to platform FAN description Set 'capability' field to FAN core platform data.. The content of 'capability' register allows to set the mapping between the drawers and tachometers. The motivation is to avoid adding a new code in the future in order to distinct between the systems types supporting a different kinds of the FAN drawers. Signed-off-by: Vadim Pasternak Reviewed-by: Hans de Goede Link: https://lore.kernel.org/r/20200923172053.26296-6-vadimp@nvidia.com Signed-off-by: Hans de Goede --- drivers/platform/x86/mlx-platform.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/platform/x86/mlx-platform.c b/drivers/platform/x86/mlx-platform.c index 04a745095c37..986ad3dda1c1 100644 --- a/drivers/platform/x86/mlx-platform.c +++ b/drivers/platform/x86/mlx-platform.c @@ -1939,6 +1939,7 @@ static struct mlxreg_core_data mlxplat_mlxcpld_default_fan_data[] = { static struct mlxreg_core_platform_data mlxplat_default_fan_data = { .data = mlxplat_mlxcpld_default_fan_data, .counter = ARRAY_SIZE(mlxplat_mlxcpld_default_fan_data), + .capability = MLXPLAT_CPLD_LPC_REG_FAN_DRW_CAP_OFFSET, }; /* Watchdog type1: hardware implementation version1 From a20b7053b5c47cd7de23288238ea4d23502f300a Mon Sep 17 00:00:00 2001 From: Ionela Voinescu Date: Thu, 24 Sep 2020 13:30:15 +0100 Subject: [PATCH 251/265] cpufreq,arm,arm64: restructure definitions of arch_set_freq_scale() Compared to other arch_* functions, arch_set_freq_scale() has an atypical weak definition that can be replaced by a strong architecture specific implementation. The more typical support for architectural functions involves defining an empty stub in a header file if the symbol is not already defined in architecture code. Some examples involve: - #define arch_scale_freq_capacity topology_get_freq_scale - #define arch_scale_freq_invariant topology_scale_freq_invariant - #define arch_scale_cpu_capacity topology_get_cpu_scale - #define arch_update_cpu_topology topology_update_cpu_topology - #define arch_scale_thermal_pressure topology_get_thermal_pressure - #define arch_set_thermal_pressure topology_set_thermal_pressure Bring arch_set_freq_scale() in line with these functions by renaming it to topology_set_freq_scale() in the arch topology driver, and by defining the arch_set_freq_scale symbol to point to the new function for arm and arm64. While there are other users of the arch_topology driver, this patch defines arch_set_freq_scale for arm and arm64 only, due to their existing definitions of arch_scale_freq_capacity. This is the getter function of the frequency invariance scale factor and without a getter function, the setter function - arch_set_freq_scale() has not purpose. Signed-off-by: Ionela Voinescu Acked-by: Viresh Kumar Acked-by: Catalin Marinas Acked-by: Sudeep Holla (BL_SWITCHER and topology parts) Signed-off-by: Rafael J. Wysocki --- arch/arm/include/asm/topology.h | 1 + arch/arm64/include/asm/topology.h | 1 + drivers/base/arch_topology.c | 4 ++-- drivers/cpufreq/cpufreq.c | 7 ------- include/linux/arch_topology.h | 2 ++ include/linux/cpufreq.h | 11 ++++++++--- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index 9219e67befbe..e5e3d5ce4d55 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -8,6 +8,7 @@ #include /* Replace task scheduler's default frequency-invariant accounting */ +#define arch_set_freq_scale topology_set_freq_scale #define arch_scale_freq_capacity topology_get_freq_scale #define arch_scale_freq_invariant topology_scale_freq_invariant diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index 7cb519473fbd..11a465243f66 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -26,6 +26,7 @@ void topology_scale_freq_tick(void); #endif /* CONFIG_ARM64_AMU_EXTN */ /* Replace task scheduler's default frequency-invariant accounting */ +#define arch_set_freq_scale topology_set_freq_scale #define arch_scale_freq_capacity topology_get_freq_scale #define arch_scale_freq_invariant topology_scale_freq_invariant diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 89cae168b076..c1a9e2fb634e 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -33,8 +33,8 @@ __weak bool arch_freq_counters_available(const struct cpumask *cpus) } DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE; -void arch_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq, - unsigned long max_freq) +void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq, + unsigned long max_freq) { unsigned long scale; int i; diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index db00693a586a..1877f5e2e5b0 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -160,13 +160,6 @@ u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) } EXPORT_SYMBOL_GPL(get_cpu_idle_time); -__weak void arch_set_freq_scale(const struct cpumask *cpus, - unsigned long cur_freq, - unsigned long max_freq) -{ -} -EXPORT_SYMBOL_GPL(arch_set_freq_scale); - /* * This is a generic cpufreq init() routine which can be used by cpufreq * drivers of SMP systems. It will do following: diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index 083df331a3c9..0f6cd6b73a61 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -30,6 +30,8 @@ static inline unsigned long topology_get_freq_scale(int cpu) return per_cpu(freq_scale, cpu); } +void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq, + unsigned long max_freq); bool topology_scale_freq_invariant(void); bool arch_freq_counters_available(const struct cpumask *cpus); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 9f779fbdbe7b..fa37b1c66443 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -1011,9 +1011,14 @@ static inline void sched_cpufreq_governor_change(struct cpufreq_policy *policy, extern void arch_freq_prepare_all(void); extern unsigned int arch_freq_get_on_cpu(int cpu); -extern void arch_set_freq_scale(const struct cpumask *cpus, - unsigned long cur_freq, - unsigned long max_freq); +#ifndef arch_set_freq_scale +static __always_inline +void arch_set_freq_scale(const struct cpumask *cpus, + unsigned long cur_freq, + unsigned long max_freq) +{ +} +#endif /* the following are really really optional */ extern struct freq_attr cpufreq_freq_attr_scaling_available_freqs; From 6699e91c071715efefd7d5709755e557e948fdd9 Mon Sep 17 00:00:00 2001 From: Ionela Voinescu Date: Thu, 24 Sep 2020 13:30:16 +0100 Subject: [PATCH 252/265] arm: disable frequency invariance for CONFIG_BL_SWITCHER big.LITTLE switching complicates the setting of a correct cpufreq-based frequency invariance scale factor due to (as observed in drivers/cpufreq/vexpress-spc-cpufreq.c): - Incorrect current and maximum frequencies as a result of the exposure of a virtual frequency table to the cpufreq core, - Missed updates as a result of asynchronous frequency adjustments caused by frequency changes in other CPU pairs. Given that its functionality is atypical in regards to frequency invariance and this is an old technology, disable frequency invariance for when big.LITTLE switching is configured in to prevent incorrect scale setting. Signed-off-by: Ionela Voinescu Suggested-by: Dietmar Eggemann Acked-by: Viresh Kumar Acked-by: Sudeep Holla Signed-off-by: Rafael J. Wysocki --- arch/arm/include/asm/topology.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index e5e3d5ce4d55..470299ee2fba 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -7,10 +7,13 @@ #include #include +/* big.LITTLE switcher is incompatible with frequency invariance */ +#ifndef CONFIG_BL_SWITCHER /* Replace task scheduler's default frequency-invariant accounting */ #define arch_set_freq_scale topology_set_freq_scale #define arch_scale_freq_capacity topology_get_freq_scale #define arch_scale_freq_invariant topology_scale_freq_invariant +#endif /* Replace task scheduler's default cpu-invariant accounting */ #define arch_scale_cpu_capacity topology_get_cpu_scale From 2ef0342530b0f487a65b5f6c94c85dce5f9b53b5 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Oct 2020 08:53:03 +0200 Subject: [PATCH 253/265] PNP: remove the now unused pnp_find_card() function All user of the pnp_find_card() compat wrapper are gone, so remove the function as well. Signed-off-by: Christoph Hellwig Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/pnp.rst | 4 ---- drivers/pnp/isapnp/compat.c | 23 ----------------------- include/linux/isapnp.h | 6 ------ 3 files changed, 33 deletions(-) diff --git a/Documentation/admin-guide/pnp.rst b/Documentation/admin-guide/pnp.rst index bab2d10631f0..3eda08191d13 100644 --- a/Documentation/admin-guide/pnp.rst +++ b/Documentation/admin-guide/pnp.rst @@ -281,10 +281,6 @@ ISAPNP drivers. They should serve as a temporary solution only. They are as follows:: - struct pnp_card *pnp_find_card(unsigned short vendor, - unsigned short device, - struct pnp_card *from) - struct pnp_dev *pnp_find_dev(struct pnp_card *card, unsigned short vendor, unsigned short function, diff --git a/drivers/pnp/isapnp/compat.c b/drivers/pnp/isapnp/compat.c index 6c845b628316..035e95092489 100644 --- a/drivers/pnp/isapnp/compat.c +++ b/drivers/pnp/isapnp/compat.c @@ -21,28 +21,6 @@ static void pnp_convert_id(char *buf, unsigned short vendor, (device >> 12) & 0x0f, (device >> 8) & 0x0f); } -struct pnp_card *pnp_find_card(unsigned short vendor, unsigned short device, - struct pnp_card *from) -{ - char id[8]; - char any[8]; - struct list_head *list; - - pnp_convert_id(id, vendor, device); - pnp_convert_id(any, ISAPNP_ANY_ID, ISAPNP_ANY_ID); - - list = from ? from->global_list.next : pnp_cards.next; - - while (list != &pnp_cards) { - struct pnp_card *card = global_to_pnp_card(list); - - if (compare_pnp_id(card->id, id) || (memcmp(id, any, 7) == 0)) - return card; - list = list->next; - } - return NULL; -} - struct pnp_dev *pnp_find_dev(struct pnp_card *card, unsigned short vendor, unsigned short function, struct pnp_dev *from) { @@ -86,5 +64,4 @@ struct pnp_dev *pnp_find_dev(struct pnp_card *card, unsigned short vendor, return NULL; } -EXPORT_SYMBOL(pnp_find_card); EXPORT_SYMBOL(pnp_find_dev); diff --git a/include/linux/isapnp.h b/include/linux/isapnp.h index 11edb2109a68..dba18c95844b 100644 --- a/include/linux/isapnp.h +++ b/include/linux/isapnp.h @@ -75,9 +75,6 @@ static inline int isapnp_proc_done(void) { return 0; } #endif /* compat */ -struct pnp_card *pnp_find_card(unsigned short vendor, - unsigned short device, - struct pnp_card *from); struct pnp_dev *pnp_find_dev(struct pnp_card *card, unsigned short vendor, unsigned short function, @@ -92,9 +89,6 @@ static inline int isapnp_cfg_end(void) { return -ENODEV; } static inline unsigned char isapnp_read_byte(unsigned char idx) { return 0xff; } static inline void isapnp_write_byte(unsigned char idx, unsigned char val) { ; } -static inline struct pnp_card *pnp_find_card(unsigned short vendor, - unsigned short device, - struct pnp_card *from) { return NULL; } static inline struct pnp_dev *pnp_find_dev(struct pnp_card *card, unsigned short vendor, unsigned short function, From 3d21044e5dbadf484232beb3c5d09f836cebb0ed Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 7 Oct 2020 19:53:55 -0700 Subject: [PATCH 254/265] ACPICA: Add support for 64 bit risc-v compilation ACPICA commit 0def627be286620c64a5d4d60ce2373f119169a9 Add 64 bit risc-v architecture. Useful for acpica tools and incorporating ACPICA into the Firmware Test Suite. Link: https://github.com/acpica/acpica/commit/0def627b Signed-off-by: Colin Ian King Signed-off-by: Bob Moore Signed-off-by: Erik Kaneda Signed-off-by: Rafael J. Wysocki --- include/acpi/platform/aclinux.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/acpi/platform/aclinux.h b/include/acpi/platform/aclinux.h index 987e2af7c335..4098ba1d6599 100644 --- a/include/acpi/platform/aclinux.h +++ b/include/acpi/platform/aclinux.h @@ -190,7 +190,8 @@ #if defined(__ia64__) || (defined(__x86_64__) && !defined(__ILP32__)) ||\ defined(__aarch64__) || defined(__PPC64__) ||\ - defined(__s390x__) + defined(__s390x__) ||\ + (defined(__riscv) && (defined(__LP64__) || defined(_LP64))) #define ACPI_MACHINE_WIDTH 64 #define COMPILER_DEPENDENT_INT64 long #define COMPILER_DEPENDENT_UINT64 unsigned long From ec360131ec27d611e5dc5e03a84d3dc47d4b9ea0 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Wed, 7 Oct 2020 19:53:56 -0700 Subject: [PATCH 255/265] ACPICA: Drop the repeated word "an" in a comment ACPICA commit 9ed2c006444d1def55bc6f08164ed5d9e809c856 Link: https://github.com/acpica/acpica/commit/9ed2c006 Signed-off-by: Randy Dunlap Signed-off-by: Erik Kaneda Signed-off-by: Bob Moore Signed-off-by: Rafael J. Wysocki --- include/acpi/actypes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/acpi/actypes.h b/include/acpi/actypes.h index d50e61384f1f..647cb11d0a0a 100644 --- a/include/acpi/actypes.h +++ b/include/acpi/actypes.h @@ -824,7 +824,7 @@ typedef u8 acpi_adr_space_type; * * Note: A Data Table region is a special type of operation region * that has its own AML opcode. However, internally, the AML - * interpreter simply creates an operation region with an an address + * interpreter simply creates an operation region with an address * space type of ACPI_ADR_SPACE_DATA_TABLE. */ #define ACPI_ADR_SPACE_DATA_TABLE (acpi_adr_space_type) 0x7E /* Internal to ACPICA only */ From 465e490d290b9670c3eef7406915facd58946244 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 7 Oct 2020 19:53:57 -0700 Subject: [PATCH 256/265] ACPICA: Tree-wide: fix various typos and spelling mistakes ACPICA commit 6648a6ac8410813bcfedb5c8345259dd155ea851 Fix spelling issues found using the codespell checker Link: https://github.com/acpica/acpica/commit/6648a6ac Signed-off-by: Colin Ian King Signed-off-by: Bob Moore Signed-off-by: Erik Kaneda Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpica/dbinput.c | 2 +- drivers/acpi/acpica/nsxfobj.c | 3 ++- include/acpi/acconfig.h | 2 +- tools/power/acpi/os_specific/service_layers/oslinuxtbl.c | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/acpi/acpica/dbinput.c b/drivers/acpi/acpica/dbinput.c index ee6a1b77af3f..568d1b0b1f5d 100644 --- a/drivers/acpi/acpica/dbinput.c +++ b/drivers/acpi/acpica/dbinput.c @@ -436,7 +436,7 @@ static void acpi_db_display_help(char *command) acpi_os_printf("\n"); } else { - /* Display help for all commands that match the subtring */ + /* Display help for all commands that match the substring */ acpi_db_display_command_info(command, TRUE); } diff --git a/drivers/acpi/acpica/nsxfobj.c b/drivers/acpi/acpica/nsxfobj.c index c022bef263e5..324269481160 100644 --- a/drivers/acpi/acpica/nsxfobj.c +++ b/drivers/acpi/acpica/nsxfobj.c @@ -24,7 +24,8 @@ ACPI_MODULE_NAME("nsxfobj") * * RETURN: Status * - * DESCRIPTION: This routine returns the type associatd with a particular handle + * DESCRIPTION: This routine returns the type associated with a particular + * handle * ******************************************************************************/ acpi_status acpi_get_type(acpi_handle handle, acpi_object_type *ret_type) diff --git a/include/acpi/acconfig.h b/include/acpi/acconfig.h index 5940a3c68a96..a225eff499c8 100644 --- a/include/acpi/acconfig.h +++ b/include/acpi/acconfig.h @@ -121,7 +121,7 @@ * *****************************************************************************/ -/* Method info (in WALK_STATE), containing local variables and argumetns */ +/* Method info (in WALK_STATE), containing local variables and arguments */ #define ACPI_METHOD_NUM_LOCALS 8 #define ACPI_METHOD_MAX_LOCAL 7 diff --git a/tools/power/acpi/os_specific/service_layers/oslinuxtbl.c b/tools/power/acpi/os_specific/service_layers/oslinuxtbl.c index dd38c2b2e1b4..11c5046dce16 100644 --- a/tools/power/acpi/os_specific/service_layers/oslinuxtbl.c +++ b/tools/power/acpi/os_specific/service_layers/oslinuxtbl.c @@ -110,7 +110,7 @@ u32 gbl_table_count = 0; * * RETURN: Status; Converted from errno. * - * DESCRIPTION: Get last errno and conver it to acpi_status. + * DESCRIPTION: Get last errno and convert it to acpi_status. * *****************************************************************************/ From 4d5840372654e33d53840b7984c853f00fcbf1eb Mon Sep 17 00:00:00 2001 From: Bob Moore Date: Wed, 7 Oct 2020 19:53:58 -0700 Subject: [PATCH 257/265] ACPICA: Add predefined names found in the SMBus sepcification Affects run-time (kernel) ACPICA, iASL, and acpi_help. The "SMBus Control Method Interface Specification, Version 1.0, December 10, 1999" containes predefined names: _SBA _SBI _SBR _SBT _SBW. This was done outside of the ACPI specification. This commit adds support for ACPICA to recognize these named objects as predefined named objects. ACPICA commit 2fe13bd7ba9f97d3bf25488bf1bb1b2329427093 Link: https://github.com/acpica/acpica/commit/2fe13bd7 Signed-off-by: Bob Moore Signed-off-by: Erik Kaneda Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpica/acpredef.h | 33 ++++++++++++++++++++++++++++++- drivers/acpi/acpica/nsarguments.c | 4 +++- drivers/acpi/acpica/psparse.c | 4 ++-- drivers/acpi/acpica/utpredef.c | 5 ++--- 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/drivers/acpi/acpica/acpredef.h b/drivers/acpi/acpica/acpredef.h index 2cbb56652f1c..57ea2276790f 100644 --- a/drivers/acpi/acpica/acpredef.h +++ b/drivers/acpi/acpica/acpredef.h @@ -101,7 +101,7 @@ enum acpi_return_package_types { /* Support macros for users of the predefined info table */ -#define METHOD_PREDEF_ARGS_MAX 4 +#define METHOD_PREDEF_ARGS_MAX 5 #define METHOD_ARG_BIT_WIDTH 3 #define METHOD_ARG_MASK 0x0007 #define ARG_COUNT_IS_MINIMUM 0x8000 @@ -117,6 +117,7 @@ enum acpi_return_package_types { #define METHOD_2ARGS(a1,a2) (2 | (a1 << 3) | (a2 << 6)) #define METHOD_3ARGS(a1,a2,a3) (3 | (a1 << 3) | (a2 << 6) | (a3 << 9)) #define METHOD_4ARGS(a1,a2,a3,a4) (4 | (a1 << 3) | (a2 << 6) | (a3 << 9) | (a4 << 12)) +#define METHOD_5ARGS(a1,a2,a3,a4,a5) (5 | (a1 << 3) | (a2 << 6) | (a3 << 9) | (a4 << 12) | (a5 << 15)) #define METHOD_RETURNS(type) (type) #define METHOD_NO_RETURN_VALUE 0 @@ -902,9 +903,39 @@ const union acpi_predefined_info acpi_gbl_predefined_methods[] = { {{"_S4W", METHOD_0ARGS, METHOD_RETURNS(ACPI_RTYPE_INTEGER)}}, + {{"_SBA", METHOD_0ARGS, + METHOD_RETURNS(ACPI_RTYPE_PACKAGE)}}, /* Fixed-length (4 Int) */ + PACKAGE_INFO(ACPI_PTYPE1_FIXED, ACPI_RTYPE_INTEGER, 4, 0, 0, 0), + + {{"_SBI", METHOD_0ARGS, + METHOD_RETURNS(ACPI_RTYPE_PACKAGE)}}, /* Fixed-length (1 Int, 1 Buf) */ + PACKAGE_INFO(ACPI_PTYPE1_FIXED, ACPI_RTYPE_INTEGER, 1, + ACPI_RTYPE_BUFFER, 1, 0), + + {{"_SBR", + METHOD_3ARGS(ACPI_TYPE_INTEGER, ACPI_TYPE_INTEGER, + ACPI_TYPE_INTEGER), + METHOD_RETURNS(ACPI_RTYPE_PACKAGE)}}, /* Fixed-length (2 Int) */ + PACKAGE_INFO(ACPI_PTYPE1_FIXED, ACPI_RTYPE_INTEGER, 2, + ACPI_RTYPE_BUFFER | ACPI_RTYPE_INTEGER, 1, 0), + {{"_SBS", METHOD_0ARGS, METHOD_RETURNS(ACPI_RTYPE_INTEGER)}}, + {{"_SBT", + METHOD_4ARGS(ACPI_TYPE_INTEGER, ACPI_TYPE_INTEGER, ACPI_TYPE_INTEGER, + ACPI_TYPE_ANY), + METHOD_RETURNS(ACPI_RTYPE_PACKAGE)}}, /* Fixed-length (2 Int, 1 Buf | Int) */ + PACKAGE_INFO(ACPI_PTYPE1_FIXED, ACPI_RTYPE_INTEGER, 2, + ACPI_RTYPE_BUFFER | ACPI_RTYPE_INTEGER, 1, 0), + + {{"_SBW", + METHOD_5ARGS(ACPI_TYPE_INTEGER, ACPI_TYPE_INTEGER, ACPI_TYPE_INTEGER, + ACPI_TYPE_INTEGER, ACPI_TYPE_ANY), + METHOD_RETURNS(ACPI_RTYPE_PACKAGE)}}, + PACKAGE_INFO(ACPI_PTYPE1_FIXED, ACPI_RTYPE_BUFFER | ACPI_RTYPE_INTEGER, + 1, 0, 0, 0), + {{"_SCP", METHOD_1ARGS(ACPI_TYPE_INTEGER) | ARG_COUNT_IS_MINIMUM, METHOD_NO_RETURN_VALUE}}, /* Acpi 1.0 allowed 1 integer arg. Acpi 3.0 expanded to 3 args. Allow both. */ diff --git a/drivers/acpi/acpica/nsarguments.c b/drivers/acpi/acpica/nsarguments.c index d5e8405e9d8f..6bbc7d350a16 100644 --- a/drivers/acpi/acpica/nsarguments.c +++ b/drivers/acpi/acpica/nsarguments.c @@ -55,7 +55,9 @@ void acpi_ns_check_argument_types(struct acpi_evaluate_info *info) arg_type = METHOD_GET_NEXT_TYPE(arg_type_list); user_arg_type = info->parameters[i]->common.type; - if (user_arg_type != arg_type) { + /* No typechecking for ACPI_TYPE_ANY */ + + if ((user_arg_type != arg_type) && (arg_type != ACPI_TYPE_ANY)) { ACPI_WARN_PREDEFINED((AE_INFO, info->full_pathname, ACPI_WARN_ALWAYS, "Argument #%u type mismatch - " diff --git a/drivers/acpi/acpica/psparse.c b/drivers/acpi/acpica/psparse.c index c780046bf294..bd3caf735be3 100644 --- a/drivers/acpi/acpica/psparse.c +++ b/drivers/acpi/acpica/psparse.c @@ -508,8 +508,8 @@ acpi_status acpi_ps_parse_aml(struct acpi_walk_state *walk_state) } /* - * If the transfer to the new method method call worked - *, a new walk state was created -- get it + * If the transfer to the new method method call worked, + * a new walk state was created -- get it */ walk_state = acpi_ds_get_current_walk_state(thread); continue; diff --git a/drivers/acpi/acpica/utpredef.c b/drivers/acpi/acpica/utpredef.c index 05fe3470fb93..dd277f7e9f10 100644 --- a/drivers/acpi/acpica/utpredef.c +++ b/drivers/acpi/acpica/utpredef.c @@ -151,7 +151,7 @@ static u32 acpi_ut_get_argument_types(char *buffer, u16 argument_types); static const char *ut_external_type_names[] = /* Indexed by ACPI_TYPE_* */ { - ", UNSUPPORTED-TYPE", + ", Type_ANY", ", Integer", ", String", ", Buffer", @@ -311,8 +311,7 @@ static u32 acpi_ut_get_argument_types(char *buffer, u16 argument_types) for (i = 0; i < arg_count; i++) { this_argument_type = METHOD_GET_NEXT_TYPE(argument_types); - if (!this_argument_type - || (this_argument_type > METHOD_MAX_ARG_TYPE)) { + if (this_argument_type > METHOD_MAX_ARG_TYPE) { printf("**** Invalid argument type (%u) " "in predefined info structure\n", this_argument_type); From 18aaa02c9b32a281832f46b9b9b3a2111db0db54 Mon Sep 17 00:00:00 2001 From: Bob Moore Date: Wed, 7 Oct 2020 19:53:59 -0700 Subject: [PATCH 258/265] ACPICA: acpi_help: Update UUID list Added the following UUID's Memory Device Generic Buttons Device NVDIMM Root Device Control Method Battery Device Graphs for _DSD method Hierarchical Data Extension ARM core_sight Graph UUID This commit squashes the following ACPICA commits: 93e9aa864a7c5e10db876a9af3f2086e50cc6cf5 759aaf73942af4a1a0d13715ce5e6b054af351fa 24751ce4cf3089a7c1709e4986bc82dfb5cca7bc Link: https://github.com/acpica/acpica/commit/93e9aa86 Link: https://github.com/acpica/acpica/commit/759aaf73 Link: https://github.com/acpica/acpica/commit/24751ce4 Signed-off-by: Bob Moore Signed-off-by: Erik Kaneda Signed-off-by: Rafael J. Wysocki --- include/acpi/acuuid.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/acpi/acuuid.h b/include/acpi/acuuid.h index 9e1367b19069..10e30a5030ee 100644 --- a/include/acpi/acuuid.h +++ b/include/acpi/acuuid.h @@ -27,6 +27,10 @@ #define UUID_PCI_HOST_BRIDGE "33db4d5b-1ff7-401c-9657-7441c03dd766" #define UUID_I2C_DEVICE "3cdff6f7-4267-4555-ad05-b30a3d8938de" #define UUID_POWER_BUTTON "dfbcf3c5-e7a5-44e6-9c1f-29c76f6e059c" +#define UUID_MEMORY_DEVICE "03b19910-f473-11dd-87af-0800200c9a66" +#define UUID_GENERIC_BUTTONS_DEVICE "fa6bd625-9ce8-470d-a2c7-b3ca36c4282e" +#define UUID_NVDIMM_ROOT_DEVICE "2f10e7a4-9e91-11e4-89d3-123b93f75cba" +#define UUID_CONTROL_METHOD_BATTERY "f18fc78b-0f15-4978-b793-53f833a1d35b" /* Interfaces */ @@ -56,5 +60,8 @@ #define UUID_BATTERY_THERMAL_LIMIT "4c2067e3-887d-475c-9720-4af1d3ed602e" #define UUID_THERMAL_EXTENSIONS "14d399cd-7a27-4b18-8fb4-7cb7b9f4e500" #define UUID_DEVICE_PROPERTIES "daffd814-6eba-4d8c-8a91-bc9bbf4aa301" +#define UUID_DEVICE_GRAPHS "ab02a46b-74c7-45a2-bd68-f7d344ef2153" +#define UUID_HIERARCHICAL_DATA_EXTENSION "dbb8e3e6-5886-4ba6-8795-1319f52a966b" +#define UUID_CORESIGHT_GRAPH "3ecbc8b6-1d0e-4fb3-8107-e627f805c6cd" #endif /* __ACUUID_H__ */ From ef3efb439aef2ad4409cba5f3d865be87fa507c4 Mon Sep 17 00:00:00 2001 From: Bob Moore Date: Wed, 7 Oct 2020 19:54:00 -0700 Subject: [PATCH 259/265] ACPICA: iASL: Return exceptions for string-to-integer conversions This allows iASL to generate errors by passing exceptions that may be encountered during string-to-integer conversions. The exceptions point out invalid hex, decimal, and octal integers. ACPICA commit e98b8c0a3d96fdabb167c0ef18a809b32ade3228 Link: https://github.com/acpica/acpica/commit/e98b8c0a Signed-off-by: Bob Moore Signed-off-by: Erik Kaneda Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpica/utstrsuppt.c | 33 +++++++++++++++++++++++--------- include/acpi/acexcep.h | 4 ++-- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/drivers/acpi/acpica/utstrsuppt.c b/drivers/acpi/acpica/utstrsuppt.c index 05ff20049b87..2d91003fcf26 100644 --- a/drivers/acpi/acpica/utstrsuppt.c +++ b/drivers/acpi/acpica/utstrsuppt.c @@ -45,10 +45,15 @@ acpi_status acpi_ut_convert_octal_string(char *string, u64 *return_value_ptr) /* Convert each ASCII byte in the input string */ while (*string) { - - /* Character must be ASCII 0-7, otherwise terminate with no error */ - + /* + * Character must be ASCII 0-7, otherwise: + * 1) Runtime: terminate with no error, per the ACPI spec + * 2) Compiler: return an error + */ if (!(ACPI_IS_OCTAL_DIGIT(*string))) { +#ifdef ACPI_ASL_COMPILER + status = AE_BAD_OCTAL_CONSTANT; +#endif break; } @@ -94,10 +99,15 @@ acpi_status acpi_ut_convert_decimal_string(char *string, u64 *return_value_ptr) /* Convert each ASCII byte in the input string */ while (*string) { - - /* Character must be ASCII 0-9, otherwise terminate with no error */ - + /* + * Character must be ASCII 0-9, otherwise: + * 1) Runtime: terminate with no error, per the ACPI spec + * 2) Compiler: return an error + */ if (!isdigit(*string)) { +#ifdef ACPI_ASL_COMPILER + status = AE_BAD_DECIMAL_CONSTANT; +#endif break; } @@ -143,10 +153,15 @@ acpi_status acpi_ut_convert_hex_string(char *string, u64 *return_value_ptr) /* Convert each ASCII byte in the input string */ while (*string) { - - /* Must be ASCII A-F, a-f, or 0-9, otherwise terminate with no error */ - + /* + * Character must be ASCII A-F, a-f, or 0-9, otherwise: + * 1) Runtime: terminate with no error, per the ACPI spec + * 2) Compiler: return an error + */ if (!isxdigit(*string)) { +#ifdef ACPI_ASL_COMPILER + status = AE_BAD_HEX_CONSTANT; +#endif break; } diff --git a/include/acpi/acexcep.h b/include/acpi/acexcep.h index 436cd1411c3a..2fc624a61769 100644 --- a/include/acpi/acexcep.h +++ b/include/acpi/acexcep.h @@ -40,12 +40,12 @@ struct acpi_exception_info { char *name; -#ifdef ACPI_HELP_APP +#if defined (ACPI_HELP_APP) || defined (ACPI_ASL_COMPILER) char *description; #endif }; -#ifdef ACPI_HELP_APP +#if defined (ACPI_HELP_APP) || defined (ACPI_ASL_COMPILER) #define EXCEP_TXT(name,description) {name, description} #else #define EXCEP_TXT(name,description) {name} From 6218ab30da72717f906576fd5e63c15b990235f1 Mon Sep 17 00:00:00 2001 From: Bob Moore Date: Wed, 7 Oct 2020 19:54:01 -0700 Subject: [PATCH 260/265] ACPICA: Debugger: Add a new command: "ALL " This command will execute/evaluate all objects with a match to the argument. ACPICA commit a1a32ec054f067d1617067e2bafb0a27a8728e07 Link: https://github.com/acpica/acpica/commit/a1a32ec0 Signed-off-by: Bob Moore Signed-off-by: Erik Kaneda Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpica/acdebug.h | 4 + drivers/acpi/acpica/dbexec.c | 39 ++++++-- drivers/acpi/acpica/dbinput.c | 12 +++ drivers/acpi/acpica/dbmethod.c | 167 ++++++++++++++++++++++++++++----- 4 files changed, 188 insertions(+), 34 deletions(-) diff --git a/drivers/acpi/acpica/acdebug.h b/drivers/acpi/acpica/acdebug.h index a676daaa2da5..f8a3abdfe250 100644 --- a/drivers/acpi/acpica/acdebug.h +++ b/drivers/acpi/acpica/acdebug.h @@ -37,12 +37,14 @@ struct acpi_db_argument_info { struct acpi_db_execute_walk { u32 count; u32 max_count; + char name_seg[ACPI_NAMESEG_SIZE + 1]; }; #define PARAM_LIST(pl) pl #define EX_NO_SINGLE_STEP 1 #define EX_SINGLE_STEP 2 +#define EX_ALL 4 /* * dbxface - external debugger interfaces @@ -124,6 +126,8 @@ void acpi_db_disassemble_aml(char *statements, union acpi_parse_object *op); void acpi_db_evaluate_predefined_names(void); +void acpi_db_evaluate_all(char *name_seg); + /* * dbnames - namespace commands */ diff --git a/drivers/acpi/acpica/dbexec.c b/drivers/acpi/acpica/dbexec.c index 4027eaab18a4..d3a9521e2dc8 100644 --- a/drivers/acpi/acpica/dbexec.c +++ b/drivers/acpi/acpica/dbexec.c @@ -86,7 +86,8 @@ void acpi_db_delete_objects(u32 count, union acpi_object *objects) * * RETURN: Status * - * DESCRIPTION: Execute a control method. + * DESCRIPTION: Execute a control method. Used to evaluate objects via the + * "EXECUTE" or "EVALUATE" commands. * ******************************************************************************/ @@ -314,11 +315,12 @@ acpi_db_execution_walk(acpi_handle obj_handle, status = acpi_evaluate_object(node, NULL, NULL, &return_obj); + acpi_gbl_method_executing = FALSE; + acpi_os_printf("Evaluation of [%4.4s] returned %s\n", acpi_ut_get_node_name(node), acpi_format_exception(status)); - acpi_gbl_method_executing = FALSE; return (AE_OK); } @@ -334,7 +336,8 @@ acpi_db_execution_walk(acpi_handle obj_handle, * RETURN: None * * DESCRIPTION: Execute a control method. Name is relative to the current - * scope. + * scope. Function used for the "EXECUTE", "EVALUATE", and + * "ALL" commands * ******************************************************************************/ @@ -372,6 +375,12 @@ acpi_db_execute(char *name, char **args, acpi_object_type *types, u32 flags) return; } + if ((flags & EX_ALL) && (strlen(name) > 4)) { + acpi_os_printf("Input name (%s) must be a 4-char NameSeg\n", + name); + return; + } + name_string = ACPI_ALLOCATE(strlen(name) + 1); if (!name_string) { return; @@ -389,13 +398,24 @@ acpi_db_execute(char *name, char **args, acpi_object_type *types, u32 flags) return; } - acpi_gbl_db_method_info.name = name_string; - acpi_gbl_db_method_info.args = args; - acpi_gbl_db_method_info.types = types; - acpi_gbl_db_method_info.flags = flags; + /* Command (ALL ) to execute all methods of a particular name */ - return_obj.pointer = NULL; - return_obj.length = ACPI_ALLOCATE_BUFFER; + else if (flags & EX_ALL) { + acpi_gbl_db_method_info.name = name_string; + return_obj.pointer = NULL; + return_obj.length = ACPI_ALLOCATE_BUFFER; + acpi_db_evaluate_all(name_string); + ACPI_FREE(name_string); + return; + } else { + acpi_gbl_db_method_info.name = name_string; + acpi_gbl_db_method_info.args = args; + acpi_gbl_db_method_info.types = types; + acpi_gbl_db_method_info.flags = flags; + + return_obj.pointer = NULL; + return_obj.length = ACPI_ALLOCATE_BUFFER; + } status = acpi_db_execute_setup(&acpi_gbl_db_method_info); if (ACPI_FAILURE(status)) { @@ -450,6 +470,7 @@ acpi_db_execute(char *name, char **args, acpi_object_type *types, u32 flags) (u32)return_obj.length); acpi_db_dump_external_object(return_obj.pointer, 1); + acpi_os_printf("\n"); /* Dump a _PLD buffer if present */ diff --git a/drivers/acpi/acpica/dbinput.c b/drivers/acpi/acpica/dbinput.c index 568d1b0b1f5d..2952856b8a67 100644 --- a/drivers/acpi/acpica/dbinput.c +++ b/drivers/acpi/acpica/dbinput.c @@ -37,6 +37,7 @@ acpi_db_match_command_help(const char *command, enum acpi_ex_debugger_commands { CMD_NOT_FOUND = 0, CMD_NULL, + CMD_ALL, CMD_ALLOCATIONS, CMD_ARGS, CMD_ARGUMENTS, @@ -115,6 +116,7 @@ enum acpi_ex_debugger_commands { static const struct acpi_db_command_info acpi_gbl_db_commands[] = { {"", 0}, {"", 0}, + {"ALL", 1}, {"ALLOCATIONS", 0}, {"ARGS", 0}, {"ARGUMENTS", 0}, @@ -222,6 +224,7 @@ static const struct acpi_db_command_help acpi_gbl_db_command_help[] = { {1, " Type ", "Display object type\n"}, {0, "\nControl Method Execution:", "\n"}, + {1, " All ", "Evaluate all objects named NameSeg\n"}, {1, " Evaluate [Arguments]", "Evaluate object or control method\n"}, {1, " Execute [Arguments]", "Synonym for Evaluate\n"}, @@ -740,6 +743,15 @@ acpi_db_command_dispatch(char *input_buffer, } break; + case CMD_ALL: + + acpi_os_printf("Executing all objects with NameSeg: %s\n", + acpi_gbl_db_args[1]); + acpi_db_execute(acpi_gbl_db_args[1], &acpi_gbl_db_args[2], + &acpi_gbl_db_arg_types[2], + EX_NO_SINGLE_STEP | EX_ALL); + break; + case CMD_ALLOCATIONS: #ifdef ACPI_DBG_TRACK_ALLOCATIONS diff --git a/drivers/acpi/acpica/dbmethod.c b/drivers/acpi/acpica/dbmethod.c index 4e48a7de7413..889d13828e49 100644 --- a/drivers/acpi/acpica/dbmethod.c +++ b/drivers/acpi/acpica/dbmethod.c @@ -21,6 +21,8 @@ static acpi_status acpi_db_walk_for_execute(acpi_handle obj_handle, u32 nesting_level, void *context, void **return_value); +static acpi_status acpi_db_evaluate_object(struct acpi_namespace_node *node); + /******************************************************************************* * * FUNCTION: acpi_db_set_method_breakpoint @@ -346,42 +348,26 @@ acpi_status acpi_db_disassemble_method(char *name) /******************************************************************************* * - * FUNCTION: acpi_db_walk_for_execute + * FUNCTION: acpi_db_evaluate_object * - * PARAMETERS: Callback from walk_namespace + * PARAMETERS: node - Namespace node for the object * * RETURN: Status * - * DESCRIPTION: Batch execution module. Currently only executes predefined - * ACPI names. + * DESCRIPTION: Main execution function for the Evaluate/Execute/All debugger + * commands. * ******************************************************************************/ -static acpi_status -acpi_db_walk_for_execute(acpi_handle obj_handle, - u32 nesting_level, void *context, void **return_value) +static acpi_status acpi_db_evaluate_object(struct acpi_namespace_node *node) { - struct acpi_namespace_node *node = - (struct acpi_namespace_node *)obj_handle; - struct acpi_db_execute_walk *info = - (struct acpi_db_execute_walk *)context; - struct acpi_buffer return_obj; - acpi_status status; char *pathname; u32 i; struct acpi_device_info *obj_info; struct acpi_object_list param_objects; union acpi_object params[ACPI_METHOD_NUM_ARGS]; - const union acpi_predefined_info *predefined; - - predefined = acpi_ut_match_predefined_method(node->name.ascii); - if (!predefined) { - return (AE_OK); - } - - if (node->type == ACPI_TYPE_LOCAL_SCOPE) { - return (AE_OK); - } + struct acpi_buffer return_obj; + acpi_status status; pathname = acpi_ns_get_external_pathname(node); if (!pathname) { @@ -390,7 +376,7 @@ acpi_db_walk_for_execute(acpi_handle obj_handle, /* Get the object info for number of method parameters */ - status = acpi_get_object_info(obj_handle, &obj_info); + status = acpi_get_object_info(node, &obj_info); if (ACPI_FAILURE(status)) { ACPI_FREE(pathname); return (status); @@ -421,14 +407,67 @@ acpi_db_walk_for_execute(acpi_handle obj_handle, acpi_gbl_method_executing = TRUE; status = acpi_evaluate_object(node, NULL, ¶m_objects, &return_obj); + acpi_gbl_method_executing = FALSE; acpi_os_printf("%-32s returned %s\n", pathname, acpi_format_exception(status)); - acpi_gbl_method_executing = FALSE; + if (return_obj.length) { + acpi_os_printf("Evaluation of %s returned object %p, " + "external buffer length %X\n", + pathname, return_obj.pointer, + (u32)return_obj.length); + + acpi_db_dump_external_object(return_obj.pointer, 1); + acpi_os_printf("\n"); + } + ACPI_FREE(pathname); /* Ignore status from method execution */ + return (AE_OK); + + /* Update count, check if we have executed enough methods */ + +} + +/******************************************************************************* + * + * FUNCTION: acpi_db_walk_for_execute + * + * PARAMETERS: Callback from walk_namespace + * + * RETURN: Status + * + * DESCRIPTION: Batch execution function. Evaluates all "predefined" objects -- + * the nameseg begins with an underscore. + * + ******************************************************************************/ + +static acpi_status +acpi_db_walk_for_execute(acpi_handle obj_handle, + u32 nesting_level, void *context, void **return_value) +{ + struct acpi_namespace_node *node = + (struct acpi_namespace_node *)obj_handle; + struct acpi_db_execute_walk *info = + (struct acpi_db_execute_walk *)context; + acpi_status status; + const union acpi_predefined_info *predefined; + + predefined = acpi_ut_match_predefined_method(node->name.ascii); + if (!predefined) { + return (AE_OK); + } + + if (node->type == ACPI_TYPE_LOCAL_SCOPE) { + return (AE_OK); + } + + acpi_db_evaluate_object(node); + + /* Ignore status from object evaluation */ + status = AE_OK; /* Update count, check if we have executed enough methods */ @@ -441,6 +480,52 @@ acpi_db_walk_for_execute(acpi_handle obj_handle, return (status); } +/******************************************************************************* + * + * FUNCTION: acpi_db_walk_for_execute_all + * + * PARAMETERS: Callback from walk_namespace + * + * RETURN: Status + * + * DESCRIPTION: Batch execution function. Evaluates all objects whose path ends + * with the nameseg "Info->NameSeg". Used for the "ALL" command. + * + ******************************************************************************/ + +static acpi_status +acpi_db_walk_for_execute_all(acpi_handle obj_handle, + u32 nesting_level, + void *context, void **return_value) +{ + struct acpi_namespace_node *node = + (struct acpi_namespace_node *)obj_handle; + struct acpi_db_execute_walk *info = + (struct acpi_db_execute_walk *)context; + acpi_status status; + + if (!ACPI_COMPARE_NAMESEG(node->name.ascii, info->name_seg)) { + return (AE_OK); + } + + if (node->type == ACPI_TYPE_LOCAL_SCOPE) { + return (AE_OK); + } + + /* Now evaluate the input object (node) */ + + acpi_db_evaluate_object(node); + + /* Ignore status from method execution */ + + status = AE_OK; + + /* Update count of executed methods/objects */ + + info->count++; + return (status); +} + /******************************************************************************* * * FUNCTION: acpi_db_evaluate_predefined_names @@ -470,3 +555,35 @@ void acpi_db_evaluate_predefined_names(void) acpi_os_printf("Evaluated %u predefined names in the namespace\n", info.count); } + +/******************************************************************************* + * + * FUNCTION: acpi_db_evaluate_all + * + * PARAMETERS: none_acpi_gbl_db_method_info + * + * RETURN: None + * + * DESCRIPTION: Namespace batch execution. Implements the "ALL" command. + * Execute all namepaths whose final nameseg matches the + * input nameseg. + * + ******************************************************************************/ + +void acpi_db_evaluate_all(char *name_seg) +{ + struct acpi_db_execute_walk info; + + info.count = 0; + info.max_count = ACPI_UINT32_MAX; + ACPI_COPY_NAMESEG(info.name_seg, name_seg); + info.name_seg[ACPI_NAMESEG_SIZE] = 0; + + /* Search all nodes in namespace */ + + (void)acpi_walk_namespace(ACPI_TYPE_ANY, ACPI_ROOT_OBJECT, + ACPI_UINT32_MAX, acpi_db_walk_for_execute_all, + NULL, (void *)&info, NULL); + + acpi_os_printf("Evaluated %u names in the namespace\n", info.count); +} From 167504a0a654837799df6ec48024cc4fada1cebb Mon Sep 17 00:00:00 2001 From: Bob Moore Date: Wed, 7 Oct 2020 19:54:02 -0700 Subject: [PATCH 261/265] ACPICA: Remove unnecessary semicolon ACPICA commit 02ffcba2af123a891eefbaed4d37780ba1e36ccc Reported by: Zou Wei. Link: https://github.com/acpica/acpica/commit/02ffcba2 Signed-off-by: Bob Moore Signed-off-by: Erik Kaneda Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpica/nsalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/acpica/nsalloc.c b/drivers/acpi/acpica/nsalloc.c index fe9b3639a87d..83d26abcf448 100644 --- a/drivers/acpi/acpica/nsalloc.c +++ b/drivers/acpi/acpica/nsalloc.c @@ -294,7 +294,7 @@ void acpi_ns_delete_children(struct acpi_namespace_node *parent_node) node_to_delete = next_node; next_node = next_node->peer; acpi_ns_delete_node(node_to_delete); - }; + } /* Clear the parent's child pointer */ From 2bfdb7b3e95b3c11cf1d1233dac7cb9c4394ec32 Mon Sep 17 00:00:00 2001 From: Bob Moore Date: Wed, 7 Oct 2020 19:54:03 -0700 Subject: [PATCH 262/265] ACPICA: Update version to 20200925 Version 20200925 ACPICA commit 70add34a9a4729b5496d0695ad14d6a1dc435879 Link: https://github.com/acpica/acpica/commit/70add34a Signed-off-by: Bob Moore Signed-off-by: Erik Kaneda Signed-off-by: Rafael J. Wysocki --- include/acpi/acpixf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/acpi/acpixf.h b/include/acpi/acpixf.h index 9dc816641286..be7de305a622 100644 --- a/include/acpi/acpixf.h +++ b/include/acpi/acpixf.h @@ -12,7 +12,7 @@ /* Current ACPICA subsystem version in YYYYMMDD format */ -#define ACPI_CA_VERSION 0x20200717 +#define ACPI_CA_VERSION 0x20200925 #include #include From 1a3f7813f38e400b0b63492b626b425f0a043d0b Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Thu, 8 Oct 2020 21:38:30 +0200 Subject: [PATCH 263/265] MAINTAINERS: update X86 PLATFORM DRIVERS entry with new kernel.org git repo The pdx86 maintainers have moved their git tree from infradead.org to kernel.org, update the X86 PLATFORM DRIVERS MAINTAINERS entry for this. Signed-off-by: Hans de Goede --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index edd2b19c240f..2a2170ff3a91 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -18861,7 +18861,7 @@ M: Hans de Goede M: Mark Gross L: platform-driver-x86@vger.kernel.org S: Maintained -T: git git://git.infradead.org/linux-platform-drivers-x86.git +T: git git://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git F: drivers/platform/olpc/ F: drivers/platform/x86/ From b7af6080a3d2d35a410db3eeb162e1ab49c4d27f Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 12 Oct 2020 10:20:07 +0530 Subject: [PATCH 264/265] cpufreq: stats: Fix string format specifier mismatch Fix following warning: drivers/cpufreq/cpufreq_stats.c:63:10: warning: %d in format string (no. 1) requires 'int' but the argument type is 'unsigned int' Fixes: 40c3bd4cfa6f ("cpufreq: stats: Defer stats update to cpufreq_stats_record_transition()") Reported-by: kernel test robot Signed-off-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq_stats.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index 1b1389face85..6cd5c8ab5d49 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -62,7 +62,7 @@ static ssize_t show_total_trans(struct cpufreq_policy *policy, char *buf) if (READ_ONCE(stats->reset_pending)) return sprintf(buf, "%d\n", 0); else - return sprintf(buf, "%d\n", stats->total_trans); + return sprintf(buf, "%u\n", stats->total_trans); } cpufreq_freq_attr_ro(total_trans); From ab0a40ea88204e1291b56da8128e2845fec8ee88 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Fri, 9 Oct 2020 14:25:23 +0200 Subject: [PATCH 265/265] perf build: Allow nested externs to enable BUILD_BUG() usage Currently the BUILD_BUG() macro is expanded to the following: do { extern void __compiletime_assert_0(void) __attribute__((error("BUILD_BUG failed"))); if (!(!(1))) __compiletime_assert_0(); } while (0); If used in a function body this would obviously produce build errors with -Wnested-externs and -Werror. To enable BUILD_BUG() usage in tools/arch/x86/lib/insn.c which perf includes in intel-pt-decoder, build perf without -Wnested-externs. Reported-by: Stephen Rothwell Tested-by: Stephen Rothwell # build tested Signed-off-by: Vasily Gorbik Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/patch-1.thread-251403.git-2514037e9477.your-ad-here.call-01602244460-ext-7088@work.hours --- tools/perf/Makefile.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index 190be4fa5c21..8137a6046a47 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -16,7 +16,7 @@ $(shell printf "" > $(OUTPUT).config-detected) detected = $(shell echo "$(1)=y" >> $(OUTPUT).config-detected) detected_var = $(shell echo "$(1)=$($(1))" >> $(OUTPUT).config-detected) -CFLAGS := $(EXTRA_CFLAGS) $(EXTRA_WARNINGS) +CFLAGS := $(EXTRA_CFLAGS) $(filter-out -Wnested-externs,$(EXTRA_WARNINGS)) include $(srctree)/tools/scripts/Makefile.arch