Repository navigation
Expand file tree
/
Copy pathvalidation.py
More file actions
256 lines (206 loc) · 9.35 KB
/
Copy pathvalidation.py
File metadata and controls
256 lines (206 loc) · 9.35 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
"""Input validation for DocsHaven."""
import os
import re
from datetime import datetime, timezone
from pathlib import Path
_MAX_QUERY_LENGTH = 10000
_MAX_CONTENT_LENGTH = 10_000_000 # the repo's content-size precedent: kb_update caps content at 10MB (server.py)
_MAX_FTS5_TOKENS = 100
_MAX_SEARCH_LIMIT = 1000
_VALID_URL_SCHEMES = ("https://", "http://", "git@")
_ALLOWED_GIT_DOMAINS = {"github.com", "gitlab.com", "bitbucket.org", "codeberg.org"}
_COLLECTION_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$")
# PEP 508 package-name grammar: letter_or_digit (letter_or_digit | '-' | '_' | '.')*
# letter_or_digit — rejects empty, "../../etc", slashes and spaces.
_PYPI_NAME_PATTERN = re.compile(r"^[A-Z0-9](?:[A-Z0-9._-]*[A-Z0-9])?$", re.IGNORECASE)
def validate_url(url: str) -> str | None:
if not url.startswith(_VALID_URL_SCHEMES):
return f"Invalid URL scheme: {url}"
if len(url) > 2048:
return "URL too long (max 2048 chars)"
dangerous_chars = set("\n\r\t`$&|;<>\\")
if any(c in url for c in dangerous_chars):
return "URL contains dangerous characters"
if "%2e" in url.lower() or "%2f" in url.lower():
return "URL contains encoded path traversal"
return _validate_url_domain(url)
def _validate_url_domain(url: str) -> str | None:
from urllib.parse import urlparse
try:
parsed = urlparse(url)
domain = parsed.hostname or ""
if domain and domain not in _ALLOWED_GIT_DOMAINS:
if url.startswith("git@"):
git_host = url.split("@")[1].split(":")[0] if "@" in url else ""
if git_host not in _ALLOWED_GIT_DOMAINS:
return f"Domain not allowed: {git_host}"
else:
return f"Domain not allowed: {domain}"
except (ValueError, AttributeError):
pass
return None
def validate_text_content(content: str) -> str | None:
"""Validate raw text content for note indexing (Storage.add_text).
Size cap reuses the repo's content-size precedent: kb_update caps its
content argument at 10MB (server.py) — the same 10_000_000 chars applies.
"""
if not content:
return "Text content cannot be empty"
if len(content) > _MAX_CONTENT_LENGTH:
return f"Text content too long (max {_MAX_CONTENT_LENGTH} chars)"
return None
def text_slug(title: str) -> str:
"""Slug for text:// note file paths: lowercase, non-alnum -> '-', trimmed.
An empty result falls back to "note" so file paths stay non-empty.
"""
slug = "".join(c if c.isalnum() else "-" for c in title.lower())
return slug.strip("-") or "note"
def domain_slug(url: str) -> str:
"""Default collection for an indexed web page (card P2-3).
The slugified netloc: lowercased, non-alnum -> '-' per the card spec
(docs.python.org -> "docs-python-org"). validate_collection rejects dots,
so every dot becomes a dash. The port is part of netloc and stays in the
slug — pages served from different ports on one host land in different
collections. An empty result falls back to "web" (validate_http_url
guarantees a host in practice).
"""
from urllib.parse import urlsplit
slug = re.sub(r"[^a-z0-9]", "-", urlsplit(url).netloc.lower())
return slug.strip("-") or "web"
def validate_pypi_name(name: str) -> str | None:
"""Validate a PyPI package name against the PEP 508 identifier grammar."""
if not name or not name.strip():
return "Package name cannot be empty"
if not _PYPI_NAME_PATTERN.match(name.strip()):
return f"Invalid PyPI package name: {name}"
return None
def pypi_slug(package_name: str) -> str:
"""Default collection for an indexed PyPI package (card P2-4).
The slugified package name: lowercased, non-alnum -> '-' (the text_slug
rule), so "Test-Pkg" -> "test-pkg". validate_collection-safe by
construction (_COLLECTION_PATTERN is [a-zA-Z0-9_-]); an empty result
falls back to "pypi" (validate_pypi_name guarantees a name in practice).
"""
slug = re.sub(r"[^a-z0-9]", "-", package_name.lower())
return slug.strip("-") or "pypi"
def validate_http_url(url: str) -> str | None:
"""Validate a bookmark URL — web links are http(s) only.
Based on validate_url's checks (scheme allowlist, 2048-char cap, dangerous
characters) minus the git-specific parts: no git@ scheme and no git-host
domain allowlist — bookmarks point at arbitrary web pages, not clone
targets. Adds a host requirement: a bare scheme is not a link. The input
is stripped and the scheme is matched case-insensitively (RFC 3986:
schemes are case-insensitive), so " https://ok.com " and
"HTTPS://EXAMPLE.COM" validate while dangerous schemes are rejected even
when spelled in a different case (JavaScript://...).
"""
url = url.strip()
if not url:
return "URL cannot be empty"
if not url.lower().startswith(("http://", "https://")):
return f"Invalid URL scheme (http/https only): {url}"
if len(url) > 2048:
return "URL too long (max 2048 chars)"
# validate_url's dangerous set minus shell-quoting metacharacters: bookmarks
# are stored, not executed — and '&' is a legitimate query-string separator.
dangerous_chars = set("\n\r\t<>")
if any(c in url for c in dangerous_chars):
return "URL contains dangerous characters"
from urllib.parse import urlparse
try:
host = urlparse(url).hostname
except ValueError:
return f"Invalid URL: {url}"
if not host:
return "URL must include a host"
return None
def normalize_http_url(url: str) -> str:
"""Fold a bookmark URL to its canonical case form (H-5, RFC 3986).
Scheme and host are case-insensitive and fold to lowercase; userinfo,
path, query and fragment keep their case. The UNIQUE(url) bookmark key
must not depend on case — https://dup.com/a and HTTPS://DUP.COM/a are one
link. Strips surrounding whitespace (the F-6 normalization step). Port
digits and IPv6 literals pass through untouched.
"""
from urllib.parse import urlsplit, urlunsplit
parts = urlsplit(url.strip())
userinfo, at, hostport = parts.netloc.rpartition("@")
host, colon, port = hostport.rpartition(":")
if colon and port.isdigit():
hostport = host.lower() + colon + port
else:
hostport = hostport.lower()
netloc = userinfo + at + hostport if at else hostport
return urlunsplit((parts.scheme.lower(), netloc, parts.path, parts.query, parts.fragment))
def validate_collection(name: str) -> str | None:
if not name:
return "Collection name cannot be empty"
if len(name) > 255:
return "Collection name too long (max 255 chars)"
if not _COLLECTION_PATTERN.match(name):
return f"Invalid collection name: {name}"
return None
def validate_query(query: str) -> str | None:
if len(query) > _MAX_QUERY_LENGTH:
return f"Query too long (max {_MAX_QUERY_LENGTH} chars)"
return None
def validate_question(question: str) -> str | None:
if not question or not question.strip():
return "Question cannot be empty"
if len(question) > _MAX_QUERY_LENGTH:
return f"Question too long (max {_MAX_QUERY_LENGTH} chars)"
return None
def validate_file_mask(mask: str) -> str | None:
if ".." in mask:
return "File mask must not contain '..' (path traversal)"
if mask.startswith("/") or mask.startswith("\\\\"):
return "File mask must not be absolute"
dangerous = set("|;&$`\x00")
if any(c in mask for c in dangerous):
return "File mask contains dangerous characters"
return None
def validate_local_dir(path: str) -> str | None:
"""Validate a local directory for indexing. Returns error or None.
The path must exist, be a directory, and be readable.
"""
if not path or not path.strip():
return "Local path cannot be empty"
if "\x00" in path:
return "Local path contains NUL byte"
p = Path(path)
if not p.exists():
return f"Local path does not exist: {path}"
if not p.is_dir():
return f"Local path is not a directory: {path}"
if not os.access(p, os.R_OK):
return f"Local path is not readable: {path}"
return None
def validate_topic(topic: str) -> str | None:
if not topic or not topic.strip():
return "Topic cannot be empty"
if len(topic) > _MAX_QUERY_LENGTH:
return f"Topic too long (max {_MAX_QUERY_LENGTH} chars)"
return None
def validate_limit(limit: int) -> str | None:
if not isinstance(limit, int) or isinstance(limit, bool):
return "Limit must be an integer"
if limit < 1:
return "Limit must be >= 1"
return None
def parse_since(since: str) -> datetime | None:
"""Parse an ISO date or datetime as naive UTC; None when unparseable.
A trailing 'Z'/'z' is stripped before parsing so Python 3.10's
fromisoformat (which rejects 'Z') accepts ISO datetimes with UTC designator.
"""
text = since[:-1] if isinstance(since, str) and since.endswith(("Z", "z")) else since
try:
parsed = datetime.fromisoformat(text)
except (TypeError, ValueError):
return None
if parsed.tzinfo is not None:
parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None)
return parsed
def validate_since(since: str) -> str | None:
if parse_since(since) is None:
return "Invalid since timestamp (expected ISO date or datetime)"
return None