Instruct search engines to not crawl archived versions (#16292)

* Instruct search engines to not crawl archived versions

* Update middleware/robots.js

Co-authored-by: Sarah Schneider <[email protected]>

* Add test

* Fix spooky bug

Co-authored-by: Chiedo <[email protected]>
Co-authored-by: Sarah Schneider <[email protected]>

Author: chiedo

middleware/robots.js

@@ -1,5 +1,6 @@
 const languages = require('../lib/languages')
 const products = require('../lib/all-products')
+const { deprecated } = require('../lib/enterprise-server-releases.js')
 
 let defaultResponse = 'User-agent: *'
 
@@ -34,5 +35,13 @@ module.exports = function (req, res, next) {
       defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}\nDisallow: /*/enterprise/*/user${product.href}`)
     })
 
+  // Disallow crawling of Deprecated enterprise versions
+  deprecated
+    .forEach(version => {
+      defaultResponse = defaultResponse
+        .concat(`\nDisallow: /*/enterprise-server@${version}/*`)
+        .concat(`\nDisallow: /*/enterprise/${version}/*`)
+    })
+
   return res.send(defaultResponse)
 }

tests/rendering/robots-txt.js

@@ -89,4 +89,25 @@ describe('robots.txt', () => {
     expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.latest}/user/actions`)).toBe(true)
     expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user/actions`)).toBe(true)
   })
+
+  it('disallows indexing of deprecated enterprise releases', async () => {
+    enterpriseServerReleases.deprecated.forEach(version => {
+      const blockedPaths = [
+        // English
+        `https://help.github.com/en/enterprise-server@${version}/actions`,
+        `https://help.github.com/en/enterprise/${version}/actions`,
+        `https://help.github.com/en/enterprise-server@${version}/actions/overview`,
+        `https://help.github.com/en/enterprise/${version}/actions/overview`,
+        // Japanese
+        `https://help.github.com/ja/enterprise-server@${version}/actions`,
+        `https://help.github.com/ja/enterprise/${version}/actions`,
+        `https://help.github.com/ja/enterprise-server@${version}/actions/overview`,
+        `https://help.github.com/ja/enterprise/${version}/actions/overview`
+      ]
+
+      blockedPaths.forEach(path => {
+        expect(robots.isAllowed(path)).toBe(false)
+      })
+    })
+  })
 })