diff --git a/CHANGES.rst b/CHANGES.rst index 004f9950..ae93f01b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,6 +3,9 @@ Changelog 5.3.1 (unreleased) ------------------ +- Add upgrade step and setuphandler to fix robots.txt + original rules adding 'Allow: /*?expand*' + [lucabel] - Nothing changed yet. diff --git a/src/redturtle/volto/profiles/default/metadata.xml b/src/redturtle/volto/profiles/default/metadata.xml index 7e649863..38ccf53b 100644 --- a/src/redturtle/volto/profiles/default/metadata.xml +++ b/src/redturtle/volto/profiles/default/metadata.xml @@ -1,6 +1,6 @@ - 4200 + 4300 profile-plone.volto:default profile-plone.app.caching:with-caching-proxy diff --git a/src/redturtle/volto/setuphandlers.py b/src/redturtle/volto/setuphandlers.py index 51247e31..0137d629 100644 --- a/src/redturtle/volto/setuphandlers.py +++ b/src/redturtle/volto/setuphandlers.py @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- from Products.CMFPlone.interfaces import INonInstallable from zope.interface import implementer +from plone import api +import logging + +logger = logging.getLogger(__name__) @implementer(INonInstallable) @@ -10,8 +14,56 @@ def getNonInstallableProfiles(self): return ["redturtle.volto:uninstall"] +def upgrade_robots_txt(context): + robots = api.portal.get_registry_record("plone.robots_txt") + lines = robots.splitlines() + + googlebot_user_agent = "User-Agent: Googlebot" + # I saw this writed also as googlebot, so: + normalized_google_bot_user_agent = googlebot_user_agent.lower().replace(" ", "") + + useragent_string = "User-Agent: " + normalized_useragent_string = useragent_string.lower().replace(" ", "") + + googlebot_allow_rule = "Allow: /*?expand*" + + googlebot_index = -1 + allow_rule_present = False + for i, line in enumerate(lines): + if line.lower().replace(" ", "") == normalized_google_bot_user_agent: + googlebot_index = i + elif ( + googlebot_index != -1 + and line.replace(" ", "").lower() # noqa + == googlebot_allow_rule.replace(" ", "").lower() # noqa + ): + allow_rule_present = True + + if googlebot_index != -1 and not allow_rule_present: + # Trova l'indice della fine della sezione User-Agent: Googlebot + end_googlebot_index = googlebot_index + 1 + while end_googlebot_index < len(lines) and not lines[ + end_googlebot_index + ].lower().replace(" ", "").startswith(normalized_useragent_string): + end_googlebot_index += 1 + + # Aggiungi Allow: /*?expand* alla fine della sezione User-Agent: Googlebot + if lines[end_googlebot_index - 1] == "": + end_googlebot_index -= 1 + lines.insert(end_googlebot_index, googlebot_allow_rule) + + lines = "\n".join(lines) + api.portal.set_registry_record("plone.robots_txt", lines) + logger.info("Upgrade robots.txt with rule for googlebot") + else: + logger.info( + "Rule for Googlebot already present in robots.txt, no action needed" + ) + + def post_install(context): """Post install script""" + upgrade_robots_txt(context) def uninstall(context): diff --git a/src/redturtle/volto/upgrades.zcml b/src/redturtle/volto/upgrades.zcml index cb841fd3..964e310e 100644 --- a/src/redturtle/volto/upgrades.zcml +++ b/src/redturtle/volto/upgrades.zcml @@ -180,5 +180,13 @@ handler=".upgrades.to_4200" /> +