diff --git a/package.json b/package.json index 278afdf6c3..3cc9ccb694 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "cookie-parser": "1.4.5", "core-js": "^3.7.0", "deepmerge": "^4.2.2", + "ejs": "^3.1.8", "express": "^4.17.1", "express-rate-limit": "^5.1.3", "fast-json-patch": "^3.0.0-1", @@ -133,6 +134,7 @@ "@ngtools/webpack": "10.2.3", "@nguniversal/builders": "~11.2.1", "@types/deep-freeze": "0.1.2", + "@types/ejs": "^3.1.1", "@types/express": "^4.17.9", "@types/file-saver": "^2.0.1", "@types/jasmine": "~3.6.0", diff --git a/server.ts b/server.ts index da3b877bc1..e084998f56 100644 --- a/server.ts +++ b/server.ts @@ -23,6 +23,7 @@ import * as pem from 'pem'; import * as https from 'https'; import * as morgan from 'morgan'; import * as express from 'express'; +import * as ejs from 'ejs'; import * as bodyParser from 'body-parser'; import * as compression from 'compression'; @@ -120,10 +121,23 @@ export function app() { })(_, (options as any), callback) ); + server.engine('ejs', ejs.renderFile); + /* * Register the view engines for html and ejs */ server.set('view engine', 'html'); + server.set('view engine', 'ejs'); + + /** + * Serve the robots.txt ejs template, filling in the origin variable + */ + server.get('/robots.txt', (req, res) => { + res.setHeader('content-type', 'text/plain'); + res.render('assets/robots.txt.ejs', { + 'origin': req.protocol + '://' + req.headers.host + }); + }); /* * Set views folder path to directory where template files are stored diff --git a/src/robots.txt b/src/robots.txt.ejs similarity index 97% rename from src/robots.txt rename to src/robots.txt.ejs index 04cbb888e0..cd5e228b3a 100644 --- a/src/robots.txt +++ b/src/robots.txt.ejs @@ -1,7 +1,7 @@ # The URL to the DSpace sitemaps # XML sitemap is listed first as it is preferred by most search engines -Sitemap: /sitemap_index.xml -Sitemap: /sitemap_index.html +Sitemap: <%= origin %>/sitemap_index.xml +Sitemap: <%= origin %>/sitemap_index.html ########################## # Default Access Group diff --git a/webpack/webpack.common.ts b/webpack/webpack.common.ts index 07e55d89d4..77fe941ec4 100644 --- a/webpack/webpack.common.ts +++ b/webpack/webpack.common.ts @@ -34,8 +34,8 @@ export const copyWebpackOptions = { }, }, { - from: path.join(__dirname, '..', 'src', 'robots.txt'), - to: 'robots.txt' + from: path.join(__dirname, '..', 'src', 'robots.txt.ejs'), + to: 'assets/robots.txt.ejs' } ] }; diff --git a/yarn.lock b/yarn.lock index 0f8ef0a6e6..941136052f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1982,6 +1982,11 @@ resolved "https://registry.yarnpkg.com/@types/deep-freeze/-/deep-freeze-0.1.2.tgz#68e5379291910e82c2f0d1629732163c2aa662cc" integrity sha512-M6x29Vk4681dght4IMnPIcF1SNmeEm0c4uatlTFhp+++H1oDK1THEIzuCC2WeCBVhX+gU0NndsseDS3zaCtlcQ== +"@types/ejs@^3.1.1": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@types/ejs/-/ejs-3.1.1.tgz#29c539826376a65e7f7d672d51301f37ed718f6d" + integrity sha512-RQul5wEfY7BjWm0sYY86cmUN/pcXWGyVxWX93DFFJvcrxax5zKlieLwA3T77xJGwNcZW0YW6CYG70p1m8xPFmA== + "@types/express-serve-static-core@^4.17.18": version "4.17.26" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.26.tgz#5d9a8eeecb9d5f9d7fc1d85f541512a84638ae88" @@ -2905,6 +2910,11 @@ async@^3.2.0: resolved "https://registry.yarnpkg.com/async/-/async-3.2.2.tgz#2eb7671034bb2194d45d30e31e24ec7e7f9670cd" integrity sha512-H0E+qZaDEfx/FY4t7iLRv1W2fFI6+pyCeTw1uN20AQPiwqwM6ojPxHxdLv4z8hi2DtnW9BOckSspLucW7pIE5g== +async@^3.2.3: + version "3.2.4" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" + integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== + asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -3222,6 +3232,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + braces@^2.3.1, braces@^2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.2.tgz#5979fd3f14cd531565e5fa2df1abfff1dfaee729" @@ -3701,7 +3718,7 @@ chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.3.0, chalk@^2.4.1, chalk@^2.4 escape-string-regexp "^1.0.5" supports-color "^5.3.0" -chalk@^4.0.0, chalk@^4.1.0, chalk@~4.1.0: +chalk@^4.0.0, chalk@^4.0.2, chalk@^4.1.0, chalk@~4.1.0: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -5511,6 +5528,13 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= +ejs@^3.1.8: + version "3.1.8" + resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.8.tgz#758d32910c78047585c7ef1f92f9ee041c1c190b" + integrity sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ== + dependencies: + jake "^10.8.5" + electron-to-chromium@^1.3.896: version "1.4.12" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.12.tgz#5f73d1278c6205fc41d7a0ebd75563046b77c5d8" @@ -6171,6 +6195,13 @@ file-uri-to-path@1.0.0: resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== +filelist@^1.0.1: + version "1.0.4" + resolved "https://registry.yarnpkg.com/filelist/-/filelist-1.0.4.tgz#f78978a1e944775ff9e62e744424f215e58352b5" + integrity sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q== + dependencies: + minimatch "^5.0.1" + filesize@^6.1.0: version "6.4.0" resolved "https://registry.yarnpkg.com/filesize/-/filesize-6.4.0.tgz#914f50471dd66fdca3cefe628bd0cde4ef769bcd" @@ -7937,6 +7968,16 @@ istanbul-reports@^3.0.2: html-escaper "^2.0.0" istanbul-lib-report "^3.0.0" +jake@^10.8.5: + version "10.8.5" + resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.5.tgz#f2183d2c59382cb274226034543b9c03b8164c46" + integrity sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw== + dependencies: + async "^3.2.3" + chalk "^4.0.2" + filelist "^1.0.1" + minimatch "^3.0.4" + jasmine-core@^3.6.0: version "3.10.1" resolved "https://registry.yarnpkg.com/jasmine-core/-/jasmine-core-3.10.1.tgz#7aa6fa2b834a522315c651a128d940eca553989a" @@ -9046,6 +9087,13 @@ minimatch@3.0.4, minimatch@^3.0.2, minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" +minimatch@^5.0.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.1.tgz#6c9dffcf9927ff2a31e74b5af11adf8b9604b022" + integrity sha512-362NP+zlprccbEt/SkxKfRMHnNY85V74mVnpUpNyr3F35covl09Kec7/sEFLt3RA4oXmewtoaanoIf67SE5Y5g== + dependencies: + brace-expansion "^2.0.1" + minimist@^1.2.0, minimist@^1.2.5: version "1.2.5" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"