robots.txt (#772)

now.json

@@ -7,6 +7,7 @@
     { "src": "^/about(.*)", "dest": "/about" },
     { "src": "^/embed(.*)", "dest": "/embed" },
     { "src": "^/_next/(.*)", "dest": "/_next/$1" },
+    { "src": "^/robots.txt", "dest": "/static/robots.txt" },
     {
       "src": "^/service-worker.js$",
       "dest": "/_next/service-worker.js",

static/robots.txt

@@ -0,0 +1,7 @@
+User-agent: *
+Allow: /$
+Allow: /about
+Allow: /static
+Allow: /embed$
+Disallow: /*
+Disallow: /embed*