Add a target to find dead link in our documentation. (#3713)

This is not part of the CI because it can be throttle and it's not super fast ~2min.

But still usefull to check that no link are down if we make some big structure changes.

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

.drone/drone.yml

@@ -12,28 +12,28 @@ workspace:
 
 steps:
 - name: test
-  image: grafana/loki-build-image:0.13.0
+  image: grafana/loki-build-image:0.14.0
   commands:
   - make BUILD_IN_CONTAINER=false test
   depends_on:
   - clone
 
 - name: lint
-  image: grafana/loki-build-image:0.13.0
+  image: grafana/loki-build-image:0.14.0
   commands:
   - make BUILD_IN_CONTAINER=false lint
   depends_on:
   - clone
 
 - name: check-generated-files
-  image: grafana/loki-build-image:0.13.0
+  image: grafana/loki-build-image:0.14.0
   commands:
   - make BUILD_IN_CONTAINER=false check-generated-files
   depends_on:
   - clone
 
 - name: check-mod
-  image: grafana/loki-build-image:0.13.0
+  image: grafana/loki-build-image:0.14.0
   commands:
   - make BUILD_IN_CONTAINER=false check-mod
   depends_on:

.lychee.toml

@@ -0,0 +1,98 @@
+###
+### Display
+###
+# Verbose program output
+verbose = false
+
+# Show progress
+progress = false
+
+
+###
+### Runtime
+###
+# Number of threads to utilize.
+# Defaults to number of cores available to the system if omitted.
+#threads = 2
+
+# Maximum number of allowed redirects
+max_redirects = 10
+
+
+###
+### Requests
+###
+# User agent to send with each request
+user_agent = "curl/7.71.1"
+
+# Website timeout from connect to response finished
+timeout = 9
+
+# Comma-separated list of accepted status codes for valid links.
+# Omit to accept all response types.
+#accept = "text/html"
+
+# Proceed for server connections considered insecure (invalid TLS)
+insecure = true
+
+# Only test links with the given scheme (e.g. https)
+# Omit to check links with any scheme
+#scheme = "https"
+
+# Request method
+method = "get"
+
+# Custom request headers
+headers = []
+
+
+###
+### Exclusions
+###
+# Exclude URLs from checking (supports regex)
+exclude = [
+    # "github", todo github token
+    "file://.*",
+    "host.docker.internal:3100",
+    "fileb://.*",
+    "ec2-user@.*.compute.amazonaws.com",
+    "querier",
+    "loki:3100",
+    "ip_or_hostname_where_loki_run:3100",
+    "dynamodb://.*",
+    "s3://.*",
+    "ec2-13-59-62-37",
+    "promtail.default",
+    "loki_addr:3100",
+    "localhost",
+    "example.com",
+    "HelloAkkaHttpServer",
+    "https://loki/",
+    "http://loki/",
+    "logs-prod-us-central1.grafana.net",
+    "ip_or_hostname_where_loki_runs",
+    "inmemory://",
+    "myloki.domain:3100",
+    "grafana/cortex-rules-action@v0.4.0",
+    "loki.git",
+    "https://github.com/grafana/website",
+    "https://github.com/settings/keys",
+]
+
+include = []
+
+# Exclude all private IPs from checking
+# Equivalent to setting `exclude_private`, `exclude_link_local`, and `exclude_loopback` to true
+exclude_all_private = false
+
+# Exclude private IP address ranges from checking
+exclude_private = false
+
+# Exclude link-local IP address range from checking
+exclude_link_local = false
+
+# Exclude loopback IP address range from checking
+exclude_loopback = true
+
+# Exclude all mail addresses from checking
+exclude_mail = false

Makefile

@@ -6,7 +6,7 @@
 .PHONY: push-images push-latest save-images load-images promtail-image loki-image build-image
 .PHONY: bigtable-backup, push-bigtable-backup
 .PHONY: benchmark-store, drone, check-mod
-.PHONY: migrate migrate-image
+.PHONY: migrate migrate-image lint-markdown
 
 SHELL = /usr/bin/env bash
 
@@ -38,7 +38,7 @@ DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES))
 # make BUILD_IN_CONTAINER=false target
 # or you can override this with an environment variable
 BUILD_IN_CONTAINER ?= true
-BUILD_IMAGE_VERSION := 0.13.0
+BUILD_IMAGE_VERSION := 0.14.0
 
 # Docker image info
 IMAGE_PREFIX ?= grafana
@@ -576,6 +576,19 @@ fmt-jsonnet:
 	@find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
 		xargs -n 1 -- jsonnetfmt -i
 
+# search for dead link in our documentation.
+# To avoid being rate limited by Github you can use an env variable GITHUB_TOKEN to pass a github token API.
+# see https://github.com/settings/tokens
+lint-markdown:
+ifeq ($(BUILD_IN_CONTAINER),true)
+	$(SUDO) docker run $(RM) $(TTY) -i \
+		-v $(shell pwd):/src/loki$(MOUNT_FLAGS) \
+		$(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@;
+else
+	lychee --verbose --config .lychee.toml ./*.md  ./docs/**/*.md  ./production/**/*.md ./cmd/**/*.md ./clients/**/*.md ./tools/**/*.md
+endif
+
+
 # usage: FUZZ_TESTCASE_PATH=/tmp/testcase make test-fuzz
 # this will run the fuzzing using /tmp/testcase and save benchmark locally.
 test-fuzz:

docs/sources/architecture/_index.md

@@ -103,7 +103,7 @@ Since all distributors share access to the same hash ring, write requests can be
 sent to any distributor.
 
 To ensure consistent query results, Loki uses
-[Dynamo-style](https://www.allthingsdistributed.com/files/amazon-dynamo-sosp2007.pdf)
+[Dynamo-style](https://www.cs.princeton.edu/courses/archive/fall15/cos518/studpres/dynamo.pdf)
 quorum consistency on reads and writes. This means that the distributor will wait
 for a positive response of at least one half plus one of the ingesters to send
 the sample to before responding to the client that initiated the send.

docs/sources/clients/promtail/configuration.md

@@ -1281,7 +1281,7 @@ positions:
   filename: /tmp/positions.yaml
 
 clients:
-  - url: http://ip_or_hostname_where_loki_runns:3100/loki/api/v1/push
+  - url: http://ip_or_hostname_where_loki_runs:3100/loki/api/v1/push
 
 scrape_configs:
   - job_name: journal

docs/sources/overview/_index.md

@@ -62,7 +62,7 @@ Since all distributors share access to the same hash ring, write requests can be
 sent to any distributor.
 
 To ensure consistent query results, Loki uses
-[Dynamo-style](https://www.allthingsdistributed.com/files/amazon-dynamo-sosp2007.pdf)
+[Dynamo-style](https://www.cs.princeton.edu/courses/archive/fall15/cos518/studpres/dynamo.pdf)
 quorum consistency on reads and writes. This means that the distributor will wait
 for a positive response of at least one half plus one of the ingesters to send
 the sample to before responding to the user.

loki-build-image/Dockerfile

@@ -7,6 +7,14 @@ RUN apk add --no-cache curl && \
     mv /tmp/linux-amd64/helm /usr/bin/helm && \
     rm -rf /tmp/linux-amd64 /tmp/helm-$HELM_VER.tgz
 
+FROM alpine as lychee
+ARG LYCHEE_VER="0.7.0"
+RUN apk add --no-cache curl && \
+    curl -L -o /tmp/lychee-$LYCHEE_VER.tgz https://github.com/lycheeverse/lychee/releases/download/${LYCHEE_VER}/lychee-${LYCHEE_VER}-x86_64-unknown-linux-gnu.tar.gz && \
+    tar -xz -C /tmp -f /tmp/lychee-$LYCHEE_VER.tgz && \
+    mv /tmp/lychee /usr/bin/lychee && \
+    rm -rf /tmp/linux-amd64 /tmp/lychee-$LYCHEE_VER.tgz
+
 FROM alpine as golangci
 RUN apk add --no-cache curl && \
     cd / && \
@@ -40,6 +48,7 @@ RUN apt-get update && \
 
 COPY --from=docker /usr/bin/docker /usr/bin/docker
 COPY --from=helm /usr/bin/helm /usr/bin/helm
+COPY --from=lychee /usr/bin/lychee /usr/bin/lychee
 COPY --from=golangci /bin/golangci-lint /usr/local/bin
 COPY --from=drone /go/bin/drone /usr/bin/drone
 COPY --from=faillint /go/bin/faillint /usr/bin/faillint