Add robots.txt support

Closes #9

Author: hyperupcall

README.md

@@ -11,7 +11,6 @@ code --install-extension install EdwinKofler.vscode-assorted-languages
 ## Language Support
 
 <!-- block-start -->
-
 - asl (`.asl`, `.dsl`)
 - asm-intel-x86-generic (`.asm`, `.nasm`, `.yasm`, `.inc`, `.s`)
 - m4 (`configure.ac`, `configure.in`, `.m4`)
@@ -65,6 +64,7 @@ code --install-extension install EdwinKofler.vscode-assorted-languages
 - textproto (`.pbtxt`, `.prototxt`, `.textproto`)
 - qlik (`.qvs`)
 - hlsl (`.sf`)
+- robots-txt
 - stylus (`.styl`, `.stylus`)
 - sxhkdrc (`.sxhkdrc.conf`)
 - systemd-unit (`.link`, `.netdev`, `.network`, `.service`, `.socket`, `.device`, `.mount`, `.automount`, `.swap`, `.target`, `.path`, `.timer`, `.snapshot`, `.slice`, `.scope`)
@@ -84,6 +84,7 @@ code --install-extension install EdwinKofler.vscode-assorted-languages
 
 ## Acknowledgements
 
+- [vscode-robots-dot-txt-support](https://github.com/BeardedFish/vscode-robots-dot-txt-support) by BeardedFish (MIT)
 - [vscode-asl](https://github.com/marysaka/vscode-asl) by marysaka (MIT)
 - [x86_64-assembly-vscode](https://github.com/13xforever/x86_64-assembly-vscode) by 13xforever (MIT)
 - [vscode_autotools](https://github.com/maelvls/vscode_autotools) by maelvls (GPL-3.0)

language-configuration/robots-txt.language-configuration.json

@@ -0,0 +1,15 @@
+{
+	"comments": {
+		"lineComment": "#"
+	},
+	"folding": {
+		"markers": {
+			"start": "^User-agent:",
+			"end": "^\\s*$"
+		}
+	},
+	"indentationRules": {
+		"increaseIndentPattern": "^User-agent:",
+		"decreaseIndentPattern": "^\\s*$"
+	}
+}

package.json

@@ -656,6 +656,14 @@
 					".sf"
 				]
 			},
+			{
+				"id": "robots-txt",
+				"filenames": [
+					"robots.txt",
+					"robots-txt"
+				],
+				"configuration": "./language-configuration/robots-txt.language-configuration.json"
+			},
 			{
 				"id": "stylus",
 				"aliases": [
@@ -1170,6 +1178,11 @@
 				"scopeName": "source.qvs",
 				"path": "./syntaxes/qlik.tmLanguage.json"
 			},
+			{
+				"language": "robots-txt",
+				"scopeName": "text.robots-txt",
+				"path": "./syntaxes/robots-txt.tmLanguage.json"
+			},
 			{
 				"language": "stylus",
 				"scopeName": "source.stylus",

syntaxes/robots-txt.tmLanguage.json

@@ -0,0 +1,57 @@
+{
+	"$schema": "https://json.schemastore.org/tmLanguage.json",
+	"name": "robots.txt",
+	"patterns": [
+		{
+			"include": "#comment"
+		},
+		{
+			"include": "#rule-block"
+		}
+	],
+	"repository": {
+		"comment": {
+			"patterns": [
+				{
+					"name": "comment.line.number-sign.robots-txt",
+					"match": "#.*$"
+				}
+			]
+		},
+		"rule-block": {
+			"patterns": [
+				{
+					"match": "(?i)^([\\s*a-zA-Z0-9-/\\\\.]+\\s*)(:)?\\s*([^\\#]*)",
+					"captures": {
+						"1": {
+							"name": "constant.language.robots-txt",
+							"patterns": [
+								{
+									"name": "invalid.illegal.directive.robots-txt",
+									"match": "(?i)^(?!\\s*(allow|crawl-delay|disallow|host|sitemap|user-agent)\\s*$).*"
+								}
+							]
+						},
+						"2": {
+							"name": "punctuation.colon.robots-txt"
+						},
+						"3": {
+							"name": "string.unquoted.value.robots-txt",
+							"patterns": [
+								{
+									"match": "-?\\d+(\\.\\d+)?",
+									"name": "constant.numeric.robots-txt"
+								},
+								{
+									"match": "true|false",
+									"name": "constant.language.robots-txt"
+								}
+							]
+						}
+					}
+				}
+			]
+		}
+	},
+	"scopeName": "text.robots-txt"
+}

testdata/robots.txt

@@ -0,0 +1,9 @@
+User-agent: *
+Disallow: /
+
+User-agent: *
+Disallow: /~joe/junk.html
+Disallow: /~joe/foo.html
+Disallow: /~joe/bar.html
+
+# this is a comment