mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-20 02:11:09 +00:00
Merge pull request #112704 from dims/drop-scripts-and-files-related-to-linkcheck
drop scripts and files related to linkcheck
This commit is contained in:
commit
ce4fd5c31d
31
LICENSES/vendor/github.com/mvdan/xurls/LICENSE
generated
vendored
31
LICENSES/vendor/github.com/mvdan/xurls/LICENSE
generated
vendored
@ -1,31 +0,0 @@
|
||||
= vendor/github.com/mvdan/xurls licensed under: =
|
||||
|
||||
Copyright (c) 2015, Daniel Martí. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
= vendor/github.com/mvdan/xurls/LICENSE 6414ed5a81af5770b80e7d632fef7125
|
@ -1,192 +0,0 @@
|
||||
/*
|
||||
Copyright 2015 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// This tool extracts the links from types.go and .md files, visits the link and
|
||||
// checks the status code of the response.
|
||||
// Usage:
|
||||
// $ linkcheck --root-dir=${ROOT}
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mvdan/xurls"
|
||||
flag "github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
var (
|
||||
rootDir = flag.String("root-dir", "", "Root directory containing documents to be processed.")
|
||||
fileSuffix = flag.StringSlice("file-suffix", []string{"types.go", ".md"}, "suffix of files to be checked")
|
||||
// URLs matching the patterns in the regWhiteList won't be checked. Patterns
|
||||
// of dummy URLs should be added to the list to avoid false alerts. Also,
|
||||
// patterns of URLs that we don't care about can be added here to improve
|
||||
// efficiency.
|
||||
regWhiteList = []*regexp.Regexp{
|
||||
regexp.MustCompile(`https://kubernetes-site\.appspot\.com`),
|
||||
// skip url that doesn't start with an English alphabet, e.g., URLs with IP addresses.
|
||||
regexp.MustCompile(`https?://[^A-Za-z].*`),
|
||||
regexp.MustCompile(`https?://localhost.*`),
|
||||
}
|
||||
// URLs listed in the fullURLWhiteList won't be checked. This separated from
|
||||
// the RegWhiteList to improve efficiency. This list includes dummy URLs that
|
||||
// are hard to be generalized by a regex, and URLs that will cause false alerts.
|
||||
fullURLWhiteList = map[string]struct{}{
|
||||
"http://github.com/some/repo.git": {},
|
||||
// This URL returns 404 when visited by this tool, but it works fine if visited by a browser.
|
||||
"http://stackoverflow.com/questions/ask?tags=kubernetes": {},
|
||||
"https://github.com/$YOUR_GITHUB_USERNAME/kubernetes.git": {},
|
||||
"https://github.com/$YOUR_GITHUB_USERNAME/kubernetes": {},
|
||||
"http://storage.googleapis.com/kubernetes-release/release/v${K8S_VERSION}/bin/darwin/amd64/kubectl": {},
|
||||
// It seems this server expects certain User-Agent value, it works fine with Chrome, but returns 404 if we issue a plain cURL to it.
|
||||
"http://supervisord.org/": {},
|
||||
"http://kubernetes.io/vX.Y/docs": {},
|
||||
"http://kubernetes.io/vX.Y/docs/": {},
|
||||
"http://kubernetes.io/vX.Y/": {},
|
||||
}
|
||||
|
||||
visitedURLs = map[string]struct{}{}
|
||||
htmlpreviewReg = regexp.MustCompile(`https://htmlpreview\.github\.io/\?`)
|
||||
httpOrhttpsReg = regexp.MustCompile(`https?.*`)
|
||||
)
|
||||
|
||||
func newWalkFunc(invalidLink *bool, client *http.Client) filepath.WalkFunc {
|
||||
return func(filePath string, info os.FileInfo, initErr error) error {
|
||||
hasSuffix := false
|
||||
for _, suffix := range *fileSuffix {
|
||||
hasSuffix = hasSuffix || strings.HasSuffix(info.Name(), suffix)
|
||||
}
|
||||
if !hasSuffix {
|
||||
return nil
|
||||
}
|
||||
|
||||
fileBytes, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
foundInvalid := false
|
||||
allURLs := xurls.Strict.FindAll(fileBytes, -1)
|
||||
fmt.Fprintf(os.Stdout, "\nChecking file %s\n", filePath)
|
||||
URL:
|
||||
for _, URL := range allURLs {
|
||||
// Don't check non http/https URL
|
||||
if !httpOrhttpsReg.Match(URL) {
|
||||
continue
|
||||
}
|
||||
for _, whiteURL := range regWhiteList {
|
||||
if whiteURL.Match(URL) {
|
||||
continue URL
|
||||
}
|
||||
}
|
||||
if _, found := fullURLWhiteList[string(URL)]; found {
|
||||
continue
|
||||
}
|
||||
// remove the htmlpreview Prefix
|
||||
processedURL := htmlpreviewReg.ReplaceAll(URL, []byte{})
|
||||
|
||||
// check if we have visited the URL.
|
||||
if _, found := visitedURLs[string(processedURL)]; found {
|
||||
continue
|
||||
}
|
||||
visitedURLs[string(processedURL)] = struct{}{}
|
||||
|
||||
retry := 0
|
||||
const maxRetry int = 3
|
||||
backoff := 100
|
||||
for retry < maxRetry {
|
||||
fmt.Fprintf(os.Stdout, "Visiting %s\n", string(processedURL))
|
||||
// Use verb HEAD to increase efficiency. However, some servers
|
||||
// do not handle HEAD well, so we need to try a GET to avoid
|
||||
// false alert.
|
||||
resp, err := client.Head(string(processedURL))
|
||||
// URLs with mock host or mock port will cause error. If we report
|
||||
// the error here, people need to add the mock URL to the white
|
||||
// list every time they add a mock URL, which will be a maintenance
|
||||
// nightmare. Hence, we decide to only report 404 to catch the
|
||||
// cases where host and port are legit, but path is not, which
|
||||
// is the most common mistake in our docs.
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
// This header is used in 301, 429 and 503.
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
||||
// And Go client will follow redirects automatically so the 301 check is probably unnecessary.
|
||||
if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable {
|
||||
retryAfter := resp.Header.Get("Retry-After")
|
||||
if seconds, err := strconv.Atoi(retryAfter); err == nil {
|
||||
backoff = seconds + 10
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "Got %d visiting %s, retry after %d seconds.\n", resp.StatusCode, string(URL), backoff)
|
||||
time.Sleep(time.Duration(backoff) * time.Second)
|
||||
backoff *= 2
|
||||
retry++
|
||||
} else if resp.StatusCode == http.StatusNotFound {
|
||||
// We only check for 404 error for now. 401, 403 errors are hard to handle.
|
||||
|
||||
// We need to try a GET to avoid false alert.
|
||||
resp, err = client.Get(string(processedURL))
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
continue URL
|
||||
}
|
||||
|
||||
foundInvalid = true
|
||||
fmt.Fprintf(os.Stderr, "Failed: in file %s, Got %d visiting %s\n", filePath, resp.StatusCode, string(URL))
|
||||
break
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
if retry == maxRetry {
|
||||
foundInvalid = true
|
||||
fmt.Fprintf(os.Stderr, "Failed: in file %s, still got 429 visiting %s after %d retries\n", filePath, string(URL), maxRetry)
|
||||
}
|
||||
}
|
||||
if foundInvalid {
|
||||
*invalidLink = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
if *rootDir == "" {
|
||||
flag.Usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
client := http.Client{
|
||||
Timeout: time.Duration(5 * time.Second),
|
||||
}
|
||||
invalidLink := false
|
||||
if err := filepath.Walk(*rootDir, newWalkFunc(&invalidLink, &client)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Fail: %v.\n", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
if invalidLink {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
2
go.mod
2
go.mod
@ -51,7 +51,6 @@ require (
|
||||
github.com/moby/ipvs v1.0.1
|
||||
github.com/mrunalp/fileutils v0.5.0
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
|
||||
github.com/mvdan/xurls v1.1.0
|
||||
github.com/onsi/ginkgo/v2 v2.2.0
|
||||
github.com/onsi/gomega v1.20.1
|
||||
github.com/opencontainers/runc v1.1.3
|
||||
@ -422,7 +421,6 @@ replace (
|
||||
github.com/morikuni/aec => github.com/morikuni/aec v1.0.0
|
||||
github.com/mrunalp/fileutils => github.com/mrunalp/fileutils v0.5.0
|
||||
github.com/munnerz/goautoneg => github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
|
||||
github.com/mvdan/xurls => github.com/mvdan/xurls v1.1.0
|
||||
github.com/mwitkow/go-conntrack => github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
|
||||
github.com/mxk/go-flowrate => github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f
|
||||
github.com/niemeyer/pretty => github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e
|
||||
|
2
go.sum
2
go.sum
@ -301,8 +301,6 @@ github.com/mrunalp/fileutils v0.5.0 h1:NKzVxiH7eSk+OQ4M+ZYW1K6h27RUV3MI6NUTsHhU6
|
||||
github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/mvdan/xurls v1.1.0 h1:OpuDelGQ1R1ueQ6sSryzi6P+1RtBpfQHM8fJwlE45ww=
|
||||
github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
|
||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
|
||||
|
@ -273,7 +273,6 @@ kube::golang::test_targets() {
|
||||
cmd/genman
|
||||
cmd/genyaml
|
||||
cmd/genswaggertypedocs
|
||||
cmd/linkcheck
|
||||
ginkgo
|
||||
test/e2e/e2e.test
|
||||
test/conformance/image/go-runner
|
||||
|
@ -32,7 +32,6 @@ source "${KUBE_ROOT}/third_party/forked/shell2junit/sh2ju.sh"
|
||||
# Excluded check patterns are always skipped.
|
||||
EXCLUDED_PATTERNS=(
|
||||
"verify-all.sh" # this script calls the make rule and would cause a loop
|
||||
"verify-linkcheck.sh" # runs in separate Jenkins job once per day due to high network usage
|
||||
"verify-*-dockerized.sh" # Don't run any scripts that intended to be run dockerized
|
||||
"verify-govet-levee.sh" # Do not run levee analysis by default while KEP-1933 implementation is in alpha.
|
||||
"verify-licenses.sh" # runs in a separate job to monitor availability of the dependencies periodically
|
||||
|
@ -1,66 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2014 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script extracts the links from types.go and .md files in pkg/api/,
|
||||
# pkg/apis/ and docs/ directories, checks the status code of the response, and
|
||||
# output the list of invalid links.
|
||||
# Usage: `hack/verify-linkcheck.sh`.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
|
||||
source "${KUBE_ROOT}/hack/lib/init.sh"
|
||||
|
||||
kube::golang::setup_env
|
||||
|
||||
make -C "${KUBE_ROOT}" WHAT=cmd/linkcheck
|
||||
|
||||
linkcheck=$(kube::util::find-binary "linkcheck")
|
||||
|
||||
kube::util::ensure-temp-dir
|
||||
OUTPUT="${KUBE_TEMP}"/linkcheck-output
|
||||
cleanup() {
|
||||
rm -rf "${OUTPUT}"
|
||||
}
|
||||
trap "cleanup" EXIT SIGINT
|
||||
mkdir -p "$OUTPUT"
|
||||
|
||||
APIROOT="${KUBE_ROOT}/pkg/api/"
|
||||
APISROOT="${KUBE_ROOT}/pkg/apis/"
|
||||
DOCROOT="${KUBE_ROOT}/docs/"
|
||||
ROOTS=("$APIROOT" "$APISROOT" "$DOCROOT")
|
||||
found_invalid=false
|
||||
for root in "${ROOTS[@]}"; do
|
||||
"${linkcheck}" "--root-dir=${root}" 2> >(tee -a "${OUTPUT}/error" >&2) && ret=0 || ret=$?
|
||||
if [[ $ret -eq 1 ]]; then
|
||||
echo "Failed: found invalid links in ${root}."
|
||||
found_invalid=true
|
||||
fi
|
||||
if [[ $ret -gt 1 ]]; then
|
||||
echo "Error running linkcheck"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ ${found_invalid} = true ]; then
|
||||
echo "Summary of invalid links:"
|
||||
cat "${OUTPUT}/error"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ex: ts=2 sw=2 et filetype=sh
|
3
vendor/github.com/mvdan/xurls/.gitignore
generated
vendored
3
vendor/github.com/mvdan/xurls/.gitignore
generated
vendored
@ -1,3 +0,0 @@
|
||||
cmd/xurls/xurls
|
||||
generate/tldsgen/tldsgen
|
||||
generate/regexgen/regexgen
|
4
vendor/github.com/mvdan/xurls/.travis.yml
generated
vendored
4
vendor/github.com/mvdan/xurls/.travis.yml
generated
vendored
@ -1,4 +0,0 @@
|
||||
language: go
|
||||
|
||||
go:
|
||||
- 1.7.x
|
27
vendor/github.com/mvdan/xurls/LICENSE
generated
vendored
27
vendor/github.com/mvdan/xurls/LICENSE
generated
vendored
@ -1,27 +0,0 @@
|
||||
Copyright (c) 2015, Daniel Martí. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
40
vendor/github.com/mvdan/xurls/README.md
generated
vendored
40
vendor/github.com/mvdan/xurls/README.md
generated
vendored
@ -1,40 +0,0 @@
|
||||
# xurls
|
||||
|
||||
[](https://godoc.org/github.com/mvdan/xurls)
|
||||
[](https://travis-ci.org/mvdan/xurls)
|
||||
|
||||
Extract urls from text using regular expressions.
|
||||
|
||||
go get -u github.com/mvdan/xurls
|
||||
|
||||
```go
|
||||
import "github.com/mvdan/xurls"
|
||||
|
||||
func main() {
|
||||
xurls.Relaxed.FindString("Do gophers live in golang.org?")
|
||||
// "golang.org"
|
||||
xurls.Strict.FindAllString("foo.com is http://foo.com/.", -1)
|
||||
// []string{"http://foo.com/"}
|
||||
}
|
||||
```
|
||||
|
||||
`Relaxed` is around five times slower than `Strict` since it does more
|
||||
work to find the URLs without relying on the scheme:
|
||||
|
||||
```
|
||||
BenchmarkStrictEmpty-4 1000000 1885 ns/op
|
||||
BenchmarkStrictSingle-4 200000 8356 ns/op
|
||||
BenchmarkStrictMany-4 100000 22547 ns/op
|
||||
BenchmarkRelaxedEmpty-4 200000 7284 ns/op
|
||||
BenchmarkRelaxedSingle-4 30000 58557 ns/op
|
||||
BenchmarkRelaxedMany-4 10000 130251 ns/op
|
||||
```
|
||||
|
||||
#### cmd/xurls
|
||||
|
||||
go get -u github.com/mvdan/xurls/cmd/xurls
|
||||
|
||||
```shell
|
||||
$ echo "Do gophers live in http://golang.org?" | xurls
|
||||
http://golang.org
|
||||
```
|
8
vendor/github.com/mvdan/xurls/regex.go
generated
vendored
8
vendor/github.com/mvdan/xurls/regex.go
generated
vendored
File diff suppressed because one or more lines are too long
17
vendor/github.com/mvdan/xurls/schemes.go
generated
vendored
17
vendor/github.com/mvdan/xurls/schemes.go
generated
vendored
@ -1,17 +0,0 @@
|
||||
// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
|
||||
// See LICENSE for licensing information
|
||||
|
||||
package xurls
|
||||
|
||||
// SchemesNoAuthority is a sorted list of some well-known url schemes that are
|
||||
// followed by ":" instead of "://". Since these are more prone to false
|
||||
// positives, we limit their matching.
|
||||
var SchemesNoAuthority = []string{
|
||||
`bitcoin`, // Bitcoin
|
||||
`file`, // Files
|
||||
`magnet`, // Torrent magnets
|
||||
`mailto`, // Mail
|
||||
`sms`, // SMS
|
||||
`tel`, // Telephone
|
||||
`xmpp`, // XMPP
|
||||
}
|
1565
vendor/github.com/mvdan/xurls/tlds.go
generated
vendored
1565
vendor/github.com/mvdan/xurls/tlds.go
generated
vendored
File diff suppressed because it is too large
Load Diff
24
vendor/github.com/mvdan/xurls/tlds_pseudo.go
generated
vendored
24
vendor/github.com/mvdan/xurls/tlds_pseudo.go
generated
vendored
@ -1,24 +0,0 @@
|
||||
// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
|
||||
// See LICENSE for licensing information
|
||||
|
||||
package xurls
|
||||
|
||||
// PseudoTLDs is a sorted list of some widely used unofficial TLDs.
|
||||
//
|
||||
// Sources:
|
||||
// * https://en.wikipedia.org/wiki/Pseudo-top-level_domain
|
||||
// * https://en.wikipedia.org/wiki/Category:Pseudo-top-level_domains
|
||||
// * https://tools.ietf.org/html/draft-grothoff-iesg-special-use-p2p-names-00
|
||||
// * https://www.iana.org/assignments/special-use-domain-names/special-use-domain-names.xhtml
|
||||
var PseudoTLDs = []string{
|
||||
`bit`, // Namecoin
|
||||
`example`, // Example domain
|
||||
`exit`, // Tor exit node
|
||||
`gnu`, // GNS by public key
|
||||
`i2p`, // I2P network
|
||||
`invalid`, // Invalid domain
|
||||
`local`, // Local network
|
||||
`localhost`, // Local network
|
||||
`test`, // Test domain
|
||||
`zkey`, // GNS domain name
|
||||
}
|
67
vendor/github.com/mvdan/xurls/xurls.go
generated
vendored
67
vendor/github.com/mvdan/xurls/xurls.go
generated
vendored
@ -1,67 +0,0 @@
|
||||
// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
|
||||
// See LICENSE for licensing information
|
||||
|
||||
// Package xurls extracts urls from plain text using regular expressions.
|
||||
package xurls
|
||||
|
||||
import "regexp"
|
||||
|
||||
//go:generate go run generate/tldsgen/main.go
|
||||
//go:generate go run generate/regexgen/main.go
|
||||
|
||||
const (
|
||||
letter = `\p{L}`
|
||||
mark = `\p{M}`
|
||||
number = `\p{N}`
|
||||
iriChar = letter + mark + number
|
||||
currency = `\p{Sc}`
|
||||
otherSymb = `\p{So}`
|
||||
endChar = iriChar + `/\-+_&~*%=#` + currency + otherSymb
|
||||
midChar = endChar + `@.,:;'?!|`
|
||||
wellParen = `\([` + midChar + `]*(\([` + midChar + `]*\)[` + midChar + `]*)*\)`
|
||||
wellBrack = `\[[` + midChar + `]*(\[[` + midChar + `]*\][` + midChar + `]*)*\]`
|
||||
wellBrace = `\{[` + midChar + `]*(\{[` + midChar + `]*\}[` + midChar + `]*)*\}`
|
||||
wellAll = wellParen + `|` + wellBrack + `|` + wellBrace
|
||||
pathCont = `([` + midChar + `]*(` + wellAll + `|[` + endChar + `])+)+`
|
||||
comScheme = `[a-zA-Z][a-zA-Z.\-+]*://`
|
||||
scheme = `(` + comScheme + `|` + otherScheme + `)`
|
||||
|
||||
iri = `[` + iriChar + `]([` + iriChar + `\-]*[` + iriChar + `])?`
|
||||
domain = `(` + iri + `\.)+`
|
||||
octet = `(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])`
|
||||
ipv4Addr = `\b` + octet + `\.` + octet + `\.` + octet + `\.` + octet + `\b`
|
||||
ipv6Addr = `([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:[0-9a-fA-F]{0,4}|:[0-9a-fA-F]{1,4})?|(:[0-9a-fA-F]{1,4}){0,2})|(:[0-9a-fA-F]{1,4}){0,3})|(:[0-9a-fA-F]{1,4}){0,4})|:(:[0-9a-fA-F]{1,4}){0,5})((:[0-9a-fA-F]{1,4}){2}|:(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])){3})|(([0-9a-fA-F]{1,4}:){1,6}|:):[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){7}:`
|
||||
ipAddr = `(` + ipv4Addr + `|` + ipv6Addr + `)`
|
||||
site = domain + gtld
|
||||
hostName = `(` + site + `|` + ipAddr + `)`
|
||||
port = `(:[0-9]*)?`
|
||||
path = `(/|/` + pathCont + `?|\b|$)`
|
||||
webURL = hostName + port + path
|
||||
|
||||
strict = `(\b` + scheme + pathCont + `)`
|
||||
relaxed = `(` + strict + `|` + webURL + `)`
|
||||
)
|
||||
|
||||
var (
|
||||
// Relaxed matches all the urls it can find.
|
||||
Relaxed = regexp.MustCompile(relaxed)
|
||||
// Strict only matches urls with a scheme to avoid false positives.
|
||||
Strict = regexp.MustCompile(strict)
|
||||
)
|
||||
|
||||
func init() {
|
||||
Relaxed.Longest()
|
||||
Strict.Longest()
|
||||
}
|
||||
|
||||
// StrictMatchingScheme produces a regexp that matches urls like Strict but
|
||||
// whose scheme matches the given regular expression.
|
||||
func StrictMatchingScheme(exp string) (*regexp.Regexp, error) {
|
||||
strictMatching := `(\b(?i)(` + exp + `)(?-i)` + pathCont + `)`
|
||||
re, err := regexp.Compile(strictMatching)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
re.Longest()
|
||||
return re, nil
|
||||
}
|
4
vendor/modules.txt
vendored
4
vendor/modules.txt
vendored
@ -570,9 +570,6 @@ github.com/mrunalp/fileutils
|
||||
# github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 => github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
|
||||
## explicit
|
||||
github.com/munnerz/goautoneg
|
||||
# github.com/mvdan/xurls v1.1.0 => github.com/mvdan/xurls v1.1.0
|
||||
## explicit
|
||||
github.com/mvdan/xurls
|
||||
# github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f => github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f
|
||||
## explicit
|
||||
github.com/mxk/go-flowrate/flowrate
|
||||
@ -2640,7 +2637,6 @@ sigs.k8s.io/yaml
|
||||
# github.com/morikuni/aec => github.com/morikuni/aec v1.0.0
|
||||
# github.com/mrunalp/fileutils => github.com/mrunalp/fileutils v0.5.0
|
||||
# github.com/munnerz/goautoneg => github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
|
||||
# github.com/mvdan/xurls => github.com/mvdan/xurls v1.1.0
|
||||
# github.com/mwitkow/go-conntrack => github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
|
||||
# github.com/mxk/go-flowrate => github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f
|
||||
# github.com/niemeyer/pretty => github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e
|
||||
|
Loading…
Reference in New Issue
Block a user