Compare commits

..

7 Commits

Author SHA1 Message Date
Daniel J Walsh
fcc57b1fa7 Merge pull request #993 from vrothberg/1.1.1
v1.1.1
2020-07-29 07:56:19 -04:00
Valentin Rothberg
13c69d48fa bump to v1.1.2-dev
Signed-off-by: Valentin Rothberg <rothberg@redhat.com>
2020-07-29 11:06:46 +02:00
Valentin Rothberg
67abbb3cef v1.1.1
* Run htpasswd from our build-container instead of registry:2
 * vendor golang.org/x/text@v0.3.3

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>
2020-07-29 11:06:18 +02:00
Miloslav Trmač
747abd054f Merge pull request #984 from mtrmac/htpasswd-1.1
Run htpasswd from our build-container instead of registry:2
2020-07-16 20:20:36 +02:00
Miloslav Trmač
8a664856cd Run htpasswd from our build-container instead of registry:2
registry:2 no longer contains htpasswd.

Also don't use log_and_run ... >> $file
because that will cause the command to be logged to $file.

Signed-off-by: Miloslav Trmač <mitr@redhat.com>
2020-07-16 19:09:51 +02:00
Daniel J Walsh
3f81ee46c3 Merge pull request #981 from vrothberg/1.1-update-x/text
[1.1] vendor golang.org/x/text@v0.3.3
2020-07-16 13:05:55 -04:00
Valentin Rothberg
14ba92b2f6 vendor golang.org/x/text@v0.3.3
Fixes: CVE-2020-14040
Signed-off-by: Valentin Rothberg <rothberg@redhat.com>
2020-07-16 11:23:08 +02:00
1881 changed files with 62305 additions and 165563 deletions

View File

@@ -1,10 +0,0 @@
version: 2
updates:
- package-ecosystem: gomod
directory: "/"
schedule:
interval: daily
time: "10:00"
timezone: Europe/Berlin
open-pull-requests-limit: 10

View File

@@ -1,209 +0,0 @@
---
# Please see contrib/<reponame>image/README.md for details on the intentions
# of this workflow.
#
# BIG FAT WARNING: This workflow is duplicated across containers/skopeo,
# containers/buildah, and containers/podman. ANY AND
# ALL CHANGES MADE HERE MUST BE MANUALLY DUPLICATED
# TO THE OTHER REPOS.
name: build multi-arch images
on:
# Upstream tends to be very active, with many merges per day.
# Only run this daily via cron schedule, or manually, not by branch push.
schedule:
- cron: '0 8 * * *'
# allows to run this workflow manually from the Actions tab
workflow_dispatch:
jobs:
multi:
name: multi-arch image build
env:
REPONAME: skopeo # No easy way to parse this out of $GITHUB_REPOSITORY
# Server/namespace value used to format FQIN
REPONAME_QUAY_REGISTRY: quay.io/skopeo
CONTAINERS_QUAY_REGISTRY: quay.io/containers
# list of architectures for build
PLATFORMS: linux/amd64,linux/s390x,linux/ppc64le,linux/arm64
# Command to execute in container to obtain project version number
VERSION_CMD: "--version" # skopeo is the entrypoint
# build several images (upstream, testing, stable) in parallel
strategy:
# By default, failure of one matrix item cancels all others
fail-fast: false
matrix:
# Builds are located under contrib/<reponame>image/<source> directory
source:
- upstream
- testing
- stable
runs-on: ubuntu-latest
# internal registry caches build for inspection before push
services:
registry:
image: quay.io/libpod/registry:2
ports:
- 5000:5000
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
with:
driver-opts: network=host
install: true
- name: Build and locally push image
uses: docker/build-push-action@v2
with:
context: contrib/${{ env.REPONAME }}image/${{ matrix.source }}
file: ./contrib/${{ env.REPONAME }}image/${{ matrix.source }}/Dockerfile
platforms: ${{ env.PLATFORMS }}
push: true
tags: localhost:5000/${{ env.REPONAME }}/${{ matrix.source }}
# Simple verification that stable images work, and
# also grab version number use in forming the FQIN.
- name: amd64 container sniff test
if: matrix.source == 'stable'
id: sniff_test
run: |
podman pull --tls-verify=false \
localhost:5000/$REPONAME/${{ matrix.source }}
VERSION_OUTPUT=$(podman run \
localhost:5000/$REPONAME/${{ matrix.source }} \
$VERSION_CMD)
echo "$VERSION_OUTPUT"
VERSION=$(awk -r -e "/^${REPONAME} version /"'{print $3}' <<<"$VERSION_OUTPUT")
test -n "$VERSION"
echo "::set-output name=version::$VERSION"
- name: Generate image FQIN(s) to push
id: reponame_reg
run: |
if [[ "${{ matrix.source }}" == 'stable' ]]; then
# The command version in image just built
VERSION='v${{ steps.sniff_test.outputs.version }}'
# workaround vim syntax-highlight bug: '
# Push both new|updated version-tag and latest-tag FQINs
FQIN="$REPONAME_QUAY_REGISTRY/stable:$VERSION,$REPONAME_QUAY_REGISTRY/stable:latest"
elif [[ "${{ matrix.source }}" == 'testing' ]]; then
# Assume some contents changed, always push latest testing.
FQIN="$REPONAME_QUAY_REGISTRY/testing:latest"
elif [[ "${{ matrix.source }}" == 'upstream' ]]; then
# Assume some contents changed, always push latest upstream.
FQIN="$REPONAME_QUAY_REGISTRY/upstream:latest"
else
echo "::error::Unknown matrix item '${{ matrix.source }}'"
exit 1
fi
echo "::warning::Pushing $FQIN"
echo "::set-output name=fqin::${FQIN}"
echo '::set-output name=push::true'
# This is substantially similar to the above logic,
# but only handles $CONTAINERS_QUAY_REGISTRY for
# the stable "latest" and named-version tagged images.
- name: Generate containers reg. image FQIN(s)
if: matrix.source == 'stable'
id: containers_reg
run: |
VERSION='v${{ steps.sniff_test.outputs.version }}'
# workaround vim syntax-highlight bug: '
# Push both new|updated version-tag and latest-tag FQINs
FQIN="$CONTAINERS_QUAY_REGISTRY/$REPONAME:$VERSION,$CONTAINERS_QUAY_REGISTRY/$REPONAME:latest"
echo "::warning::Pushing $FQIN"
echo "::set-output name=fqin::${FQIN}"
echo '::set-output name=push::true'
- name: Define LABELS multi-line env. var. value
run: |
# This is a really hacky/strange workflow idiom, required
# for setting multi-line $LABELS value for consumption in
# a future step. There is literally no cleaner way to do this :<
# https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#multiline-strings
function set_labels() {
echo 'LABELS<<DELIMITER' >> "$GITHUB_ENV"
for line; do
echo "$line" | tee -a "$GITHUB_ENV"
done
echo "DELIMITER" >> "$GITHUB_ENV"
}
declare -a lines
lines=(\
"org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}.git"
"org.opencontainers.image.revision=${GITHUB_SHA}"
"org.opencontainers.image.created=$(date -u --iso-8601=seconds)"
)
# Only the 'stable' matrix source obtains $VERSION
if [[ "${{ matrix.source }}" == "stable" ]]; then
lines+=(\
"org.opencontainers.image.version=${{ steps.sniff_test.outputs.version }}"
)
fi
set_labels "${lines[@]}"
# Separate steps to login and push for $REPONAME_QUAY_REGISTRY and
# $CONTAINERS_QUAY_REGISTRY are required, because 2 sets of credentials
# are used and namespaced within the registry. At the same time, reuse
# of non-shell steps is not supported by Github Actions nor are YAML
# anchors/aliases, nor composite actions.
# Push to $REPONAME_QUAY_REGISTRY for stable, testing. and upstream
- name: Login to ${{ env.REPONAME_QUAY_REGISTRY }}
uses: docker/login-action@v1
if: steps.reponame_reg.outputs.push == 'true'
with:
registry: ${{ env.REPONAME_QUAY_REGISTRY }}
# N/B: Secrets are not passed to workflows that are triggered
# by a pull request from a fork
username: ${{ secrets.REPONAME_QUAY_USERNAME }}
password: ${{ secrets.REPONAME_QUAY_PASSWORD }}
- name: Push images to ${{ steps.reponame_reg.outputs.fqin }}
uses: docker/build-push-action@v2
if: steps.reponame_reg.outputs.push == 'true'
with:
cache-from: type=registry,ref=localhost:5000/${{ env.REPONAME }}/${{ matrix.source }}
cache-to: type=inline
context: contrib/${{ env.REPONAME }}image/${{ matrix.source }}
file: ./contrib/${{ env.REPONAME }}image/${{ matrix.source }}/Dockerfile
platforms: ${{ env.PLATFORMS }}
push: true
tags: ${{ steps.reponame_reg.outputs.fqin }}
labels: |
${{ env.LABELS }}
# Push to $CONTAINERS_QUAY_REGISTRY only stable
- name: Login to ${{ env.CONTAINERS_QUAY_REGISTRY }}
if: steps.containers_reg.outputs.push == 'true'
uses: docker/login-action@v1
with:
registry: ${{ env.CONTAINERS_QUAY_REGISTRY}}
username: ${{ secrets.CONTAINERS_QUAY_USERNAME }}
password: ${{ secrets.CONTAINERS_QUAY_PASSWORD }}
- name: Push images to ${{ steps.containers_reg.outputs.fqin }}
if: steps.containers_reg.outputs.push == 'true'
uses: docker/build-push-action@v2
with:
cache-from: type=registry,ref=localhost:5000/${{ env.REPONAME }}/${{ matrix.source }}
cache-to: type=inline
context: contrib/${{ env.REPONAME }}image/${{ matrix.source }}
file: ./contrib/${{ env.REPONAME }}image/${{ matrix.source }}/Dockerfile
platforms: ${{ env.PLATFORMS }}
push: true
tags: ${{ steps.containers_reg.outputs.fqin }}
labels: |
${{ env.LABELS }}

View File

@@ -1,25 +0,0 @@
name: Mark stale issues and pull requests
# Please refer to https://github.com/actions/stale/blob/master/action.yml
# to see all config knobs of the stale action.
on:
schedule:
- cron: "0 0 * * *"
jobs:
stale:
runs-on: ubuntu-latest
steps:
- uses: actions/stale@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
stale-issue-message: 'A friendly reminder that this issue had no activity for 30 days.'
stale-pr-message: 'A friendly reminder that this PR had no activity for 30 days.'
stale-issue-label: 'stale-issue'
stale-pr-label: 'stale-pr'
days-before-stale: 30
days-before-close: 365
remove-stale-when-updated: true

6
.gitignore vendored
View File

@@ -1,10 +1,6 @@
*.1
/layers-*
/skopeo
result
# ignore JetBrains IDEs (GoLand) config folder
.idea
# Ignore the bin directory
bin
.idea

28
.travis.yml Normal file
View File

@@ -0,0 +1,28 @@
language: go
matrix:
include:
- os: linux
sudo: required
services:
- docker
- os: osx
go:
- 1.13.x
notifications:
email: false
install:
# NOTE: The (brew update) should not be necessary, and slows things down;
# we include it as a workaround for https://github.com/Homebrew/brew/issues/3299
# ideally Travis should bake the (brew update) into its images
# (https://github.com/travis-ci/travis-ci/issues/8552 ), but thats only going
# to happen around November 2017 per https://blog.travis-ci.com/2017-10-16-a-new-default-os-x-image-is-coming .
# Remove the (brew update) at that time.
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update && brew install gpgme ; fi
script:
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then hack/travis_osx.sh ; fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make vendor && ./hack/tree_status.sh && make check ; fi

View File

@@ -1,3 +1,3 @@
## The skopeo Project Community Code of Conduct
The skopeo project follows the [Containers Community Code of Conduct](https://github.com/containers/common/blob/main/CODE-OF-CONDUCT.md).
The skopeo project follows the [Containers Community Code of Conduct](https://github.com/containers/common/blob/master/CODE-OF-CONDUCT.md).

View File

@@ -117,34 +117,32 @@ commit automatically with `git commit -s`.
### Dependencies management
Dependencies are managed via [standard go modules](https://golang.org/ref/mod).
Make sure [`vndr`](https://github.com/LK4D4/vndr) is installed.
In order to add a new dependency to this project:
- use `go get -d path/to/dep@version` to add a new line to `go.mod`
- add a new line to `vendor.conf` according to `vndr` rules (e.g. `github.com/pkg/errors master`)
- run `make vendor`
In order to update an existing dependency:
- use `go get -d -u path/to/dep@version` to update the relevant dependency line in `go.mod`
- update the relevant dependency line in `vendor.conf`
- run `make vendor`
When new PRs for [containers/image](https://github.com/containers/image) break `skopeo` (i.e. `containers/image` tests fail in `make test-skopeo`):
- create out a new branch in your `skopeo` checkout and switch to it
- find out the version of `containers/image` you want to use and note its commit ID. You might also want to use a fork of `containers/image`, in that case note its repo
- use `go get -d github.com/$REPO/image/v5@$COMMIT_ID` to download the right version. The command will fetch the dependency and then fail because of a conflict in `go.mod`, this is expected. Note the pseudo-version (eg. `v5.13.1-0.20210707123201-50afbf0a326`)
- use `go mod edit -replace=github.com/containers/image/v5=github.com/$REPO/image/v5@$PSEUDO_VERSION` to add a replacement line to `go.mod` (e.g. `replace github.com/containers/image/v5 => github.com/moio/image/v5 v5.13.1-0.20210707123201-50afbf0a3262`)
- update `vendor.conf`. Find out the `containers/image` dependency; update it to vendor from your own branch and your own repository fork (e.g. `github.com/containers/image my-branch https://github.com/runcom/image`)
- run `make vendor`
- make any other necessary changes in the skopeo repo (e.g. add other dependencies now required by `containers/image`, or update skopeo for changed `containers/image` API)
- make any other necessary changes in the skopeo repo (e.g. add other dependencies now requied by `containers/image`, or update skopeo for changed `containers/image` API)
- optionally add new integration tests to the skopeo repo
- submit the resulting branch as a skopeo PR, marked “DO NOT MERGE”
- iterate until tests pass and the PR is reviewed
- then the original `containers/image` PR can be merged, disregarding its `make test-skopeo` failure
- as soon as possible after that, in the skopeo PR, use `go mod edit -dropreplace=github.com/containers/image` to remove the `replace` line in `go.mod`
- as soon as possible after that, in the skopeo PR, restore the `containers/image` line in `vendor.conf` to use `containers/image:master`
- run `make vendor`
- update the skopeo PR with the result, drop the “DO NOT MERGE” marking
- after tests complete successfully again, merge the skopeo PR
- after tests complete succcesfully again, merge the skopeo PR
## Communications

View File

@@ -1,4 +1,4 @@
FROM registry.fedoraproject.org/fedora:latest
FROM fedora
RUN dnf -y update && dnf install -y make git golang golang-github-cpuguy83-md2man \
# storage deps
@@ -13,7 +13,6 @@ RUN dnf -y update && dnf install -y make git golang golang-github-cpuguy83-md2ma
which tar wget hostname util-linux bsdtar socat ethtool device-mapper iptables tree findutils nmap-ncat e2fsprogs xfsprogs lsof docker iproute \
bats jq podman runc \
golint \
openssl \
&& dnf clean all
# Install two versions of the registry. The first is an older version that
@@ -21,7 +20,6 @@ RUN dnf -y update && dnf install -y make git golang golang-github-cpuguy83-md2ma
# both. This allows integration-cli tests to cover push/pull with both schema1
# and schema2 manifests.
RUN set -x \
&& export GO111MODULE=off \
&& REGISTRY_COMMIT_SCHEMA1=ec87e9b6971d831f0eff752ddb54fb64693e51cd \
&& REGISTRY_COMMIT=47a064d4195a9b56133891bbb13620c3ac83a827 \
&& export GOPATH="$(mktemp -d)" \
@@ -35,7 +33,6 @@ RUN set -x \
&& rm -rf "$GOPATH"
RUN set -x \
&& export GO111MODULE=off \
&& export GOPATH=$(mktemp -d) \
&& git clone --depth 1 -b v1.5.0-alpha.3 git://github.com/openshift/origin "$GOPATH/src/github.com/openshift/origin" \
# The sed edits out a "go < 1.5" check which works incorrectly with go ≥ 1.10. \
@@ -48,7 +45,6 @@ RUN set -x \
ENV GOPATH /usr/share/gocode:/go
ENV PATH $GOPATH/bin:/usr/share/gocode/bin:$PATH
ENV container_magic 85531765-346b-4316-bdb8-358e4cca9e5d
RUN go version
WORKDIR /go/src/github.com/containers/skopeo
COPY . /go/src/github.com/containers/skopeo

View File

@@ -1,12 +1,13 @@
FROM registry.fedoraproject.org/fedora:33
FROM ubuntu:19.10
RUN dnf update -y && \
dnf install -y \
btrfs-progs-devel \
device-mapper-devel \
golang \
gpgme-devel \
make
RUN apt-get update && apt-get install -y \
golang \
libbtrfs-dev \
git-core \
libdevmapper-dev \
libgpgme11-dev \
go-md2man \
libglib2.0-dev
ENV GOPATH=/
WORKDIR /src/github.com/containers/skopeo

143
Makefile
View File

@@ -2,37 +2,33 @@
export GOPROXY=https://proxy.golang.org
# On some plaforms (eg. macOS, FreeBSD) gpgme is installed in /usr/local/ but /usr/local/include/ is
# not in the default search path. Rather than hard-code this directory, use gpgme-config.
# Sadly that must be done at the top-level user instead of locally in the gpgme subpackage, because cgo
# supports only pkg-config, not general shell scripts, and gpgme does not install a pkg-config file.
ifeq ($(shell uname),Darwin)
PREFIX ?= ${DESTDIR}/usr/local
DARWIN_BUILD_TAG=
# On macOS, (brew install gpgme) installs it within /usr/local, but /usr/local/include is not in the default search path.
# Rather than hard-code this directory, use gpgme-config. Sadly that must be done at the top-level user
# instead of locally in the gpgme subpackage, because cgo supports only pkg-config, not general shell scripts,
# and gpgme does not install a pkg-config file.
# If gpgme is not installed or gpgme-config cant be found for other reasons, the error is silently ignored
# (and the user will probably find out because the cgo compilation will fail).
GPGME_ENV := CGO_CFLAGS="$(shell gpgme-config --cflags 2>/dev/null)" CGO_LDFLAGS="$(shell gpgme-config --libs 2>/dev/null)"
else
PREFIX ?= ${DESTDIR}/usr
endif
# The following variables very roughly follow https://www.gnu.org/prep/standards/standards.html#Makefile-Conventions .
DESTDIR ?=
PREFIX ?= /usr/local
CONTAINERSCONFDIR ?= /etc/containers
REGISTRIESDDIR ?= ${CONTAINERSCONFDIR}/registries.d
SIGSTOREDIR ?= /var/lib/containers/sigstore
BINDIR ?= ${PREFIX}/bin
MANDIR ?= ${PREFIX}/share/man
BASHCOMPLETIONSDIR ?= ${PREFIX}/share/bash-completion/completions
INSTALLDIR=${PREFIX}/bin
MANINSTALLDIR=${PREFIX}/share/man
CONTAINERSSYSCONFIGDIR=${DESTDIR}/etc/containers
REGISTRIESDDIR=${CONTAINERSSYSCONFIGDIR}/registries.d
SIGSTOREDIR=${DESTDIR}/var/lib/containers/sigstore
BASHINSTALLDIR=${PREFIX}/share/bash-completion/completions
GO ?= go
GOBIN := $(shell $(GO) env GOBIN)
GOOS ?= $(shell go env GOOS)
GOARCH ?= $(shell go env GOARCH)
ifeq ($(GOBIN),)
GOBIN := $(GOPATH)/bin
endif
# Required for integration-tests to detect they are running inside a specific
# container image. Env. var defined in image, make does not automatically
# pass to children unless explicitly exported
export container_magic
CONTAINER_RUNTIME := $(shell command -v podman 2> /dev/null || echo docker)
GOMD2MAN ?= $(shell command -v go-md2man || echo '$(GOBIN)/go-md2man')
@@ -47,10 +43,8 @@ ifeq ($(DEBUG), 1)
override GOGCFLAGS += -N -l
endif
ifeq ($(GOOS), linux)
ifneq ($(GOARCH),$(filter $(GOARCH),mips mipsle mips64 mips64le ppc64 riscv64))
GO_DYN_FLAGS="-buildmode=pie"
endif
ifeq ($(shell $(GO) env GOOS), linux)
GO_DYN_FLAGS="-buildmode=pie"
endif
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
@@ -68,15 +62,12 @@ CONTAINER_RUN := $(CONTAINER_CMD) "$(IMAGE)"
GIT_COMMIT := $(shell git rev-parse HEAD 2> /dev/null || true)
EXTRA_LDFLAGS ?=
SKOPEO_LDFLAGS := -ldflags '-X main.gitCommit=${GIT_COMMIT} $(EXTRA_LDFLAGS)'
MANPAGES_MD = $(wildcard docs/*.md)
MANPAGES ?= $(MANPAGES_MD:%.md=%)
BTRFS_BUILD_TAG = $(shell hack/btrfs_tag.sh) $(shell hack/btrfs_installed_tag.sh)
LIBDM_BUILD_TAG = $(shell hack/libdm_tag.sh)
LOCAL_BUILD_TAGS = $(BTRFS_BUILD_TAG) $(LIBDM_BUILD_TAG)
LOCAL_BUILD_TAGS = $(BTRFS_BUILD_TAG) $(LIBDM_BUILD_TAG) $(DARWIN_BUILD_TAG)
BUILDTAGS += $(LOCAL_BUILD_TAGS)
ifeq ($(DISABLE_CGO), 1)
@@ -87,17 +78,14 @@ endif
# Note: Uses the -N -l go compiler options to disable compiler optimizations
# and inlining. Using these build options allows you to subsequently
# use source debugging tools like delve.
all: bin/skopeo docs
all: binary docs-in-container
help:
@echo "Usage: make <target>"
@echo
@echo "Defaults to building bin/skopeo and docs"
@echo
@echo " * 'install' - Install binaries and documents to system locations"
@echo " * 'binary' - Build skopeo with a container"
@echo " * 'static' - Build statically linked binary"
@echo " * 'bin/skopeo' - Build skopeo locally"
@echo " * 'binary-local' - Build skopeo locally"
@echo " * 'test-unit' - Execute unit tests"
@echo " * 'test-integration' - Execute integration tests"
@echo " * 'validate' - Verify whether there is no conflict and all Go source files have been formatted, linted and vetted"
@@ -110,39 +98,25 @@ help:
binary: cmd/skopeo
${CONTAINER_RUNTIME} build ${BUILD_ARGS} -f Dockerfile.build -t skopeobuildimage .
${CONTAINER_RUNTIME} run --rm --security-opt label=disable -v $$(pwd):/src/github.com/containers/skopeo \
skopeobuildimage make bin/skopeo $(if $(DEBUG),DEBUG=$(DEBUG)) BUILDTAGS='$(BUILDTAGS)'
skopeobuildimage make binary-local $(if $(DEBUG),DEBUG=$(DEBUG)) BUILDTAGS='$(BUILDTAGS)'
# Update nix/nixpkgs.json its latest stable commit
.PHONY: nixpkgs
nixpkgs:
@nix run \
-f channel:nixos-21.05 nix-prefetch-git \
-c nix-prefetch-git \
--no-deepClone \
https://github.com/nixos/nixpkgs refs/heads/nixos-21.05 > nix/nixpkgs.json
# Build statically linked binary
.PHONY: static
static:
@nix build -f nix/
mkdir -p ./bin
cp -rfp ./result/bin/* ./bin/
binary-static: cmd/skopeo
${CONTAINER_RUNTIME} build ${BUILD_ARGS} -f Dockerfile.build -t skopeobuildimage .
${CONTAINER_RUNTIME} run --rm --security-opt label=disable -v $$(pwd):/src/github.com/containers/skopeo \
skopeobuildimage make binary-local-static $(if $(DEBUG),DEBUG=$(DEBUG)) BUILDTAGS='$(BUILDTAGS)'
# Build w/o using containers
.PHONY: bin/skopeo
bin/skopeo:
$(GPGME_ENV) $(GO) build $(MOD_VENDOR) ${GO_DYN_FLAGS} ${SKOPEO_LDFLAGS} -gcflags "$(GOGCFLAGS)" -tags "$(BUILDTAGS)" -o $@ ./cmd/skopeo
bin/skopeo.%:
GOOS=$(word 2,$(subst ., ,$@)) GOARCH=$(word 3,$(subst ., ,$@)) $(GO) build $(MOD_VENDOR) ${SKOPEO_LDFLAGS} -tags "containers_image_openpgp $(BUILDTAGS)" -o $@ ./cmd/skopeo
local-cross: bin/skopeo.darwin.amd64 bin/skopeo.linux.arm bin/skopeo.linux.arm64 bin/skopeo.windows.386.exe bin/skopeo.windows.amd64.exe
binary-local:
$(GPGME_ENV) $(GO) build $(MOD_VENDOR) ${GO_DYN_FLAGS} -ldflags "-X main.gitCommit=${GIT_COMMIT}" -gcflags "$(GOGCFLAGS)" -tags "$(BUILDTAGS)" -o skopeo ./cmd/skopeo
binary-local-static:
$(GPGME_ENV) $(GO) build $(MOD_VENDOR) -ldflags "-extldflags \"-static\" -X main.gitCommit=${GIT_COMMIT}" -gcflags "$(GOGCFLAGS)" -tags "$(BUILDTAGS)" -o skopeo ./cmd/skopeo
build-container:
${CONTAINER_RUNTIME} build ${BUILD_ARGS} -t "$(IMAGE)" .
$(MANPAGES): %: %.md
ifneq ($(DISABLE_DOCS), 1)
sed -e 's/\((skopeo.*\.md)\)//' -e 's/\[\(skopeo.*\)\]/\1/' $< | $(GOMD2MAN) -in /dev/stdin -out $@
endif
@sed -e 's/\((skopeo.*\.md)\)//' -e 's/\[\(skopeo.*\)\]/\1/' $< | $(GOMD2MAN) -in /dev/stdin -out $@
docs: $(MANPAGES)
@@ -152,28 +126,26 @@ docs-in-container:
skopeobuildimage make docs $(if $(DEBUG),DEBUG=$(DEBUG)) BUILDTAGS='$(BUILDTAGS)'
clean:
rm -rf bin docs/*.1
rm -f skopeo docs/*.1
install: install-binary install-docs install-completions
install -d -m 755 ${DESTDIR}${SIGSTOREDIR}
install -d -m 755 ${DESTDIR}${CONTAINERSCONFDIR}
install -m 644 default-policy.json ${DESTDIR}${CONTAINERSCONFDIR}/policy.json
install -d -m 755 ${DESTDIR}${REGISTRIESDDIR}
install -m 644 default.yaml ${DESTDIR}${REGISTRIESDDIR}/default.yaml
install -d -m 755 ${SIGSTOREDIR}
install -d -m 755 ${CONTAINERSSYSCONFIGDIR}
install -m 644 default-policy.json ${CONTAINERSSYSCONFIGDIR}/policy.json
install -d -m 755 ${REGISTRIESDDIR}
install -m 644 default.yaml ${REGISTRIESDDIR}/default.yaml
install-binary: bin/skopeo
install -d -m 755 ${DESTDIR}${BINDIR}
install -m 755 bin/skopeo ${DESTDIR}${BINDIR}/skopeo
install-binary: ./skopeo
install -d -m 755 ${INSTALLDIR}
install -m 755 skopeo ${INSTALLDIR}/skopeo
install-docs: docs
ifneq ($(DISABLE_DOCS), 1)
install -d -m 755 ${DESTDIR}${MANDIR}/man1
install -m 644 docs/*.1 ${DESTDIR}${MANDIR}/man1
endif
install -d -m 755 ${MANINSTALLDIR}/man1
install -m 644 docs/*.1 ${MANINSTALLDIR}/man1/
install-completions:
install -m 755 -d ${DESTDIR}${BASHCOMPLETIONSDIR}
install -m 644 completions/bash/skopeo ${DESTDIR}${BASHCOMPLETIONSDIR}/skopeo
install -m 755 -d ${BASHINSTALLDIR}
install -m 644 completions/bash/skopeo ${BASHINSTALLDIR}/skopeo
shell: build-container
$(CONTAINER_RUN) bash
@@ -182,11 +154,7 @@ check: validate test-unit test-integration test-system
# The tests can run out of entropy and block in containers, so replace /dev/random.
test-integration: build-container
$(CONTAINER_RUN) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; SKOPEO_CONTAINER_TESTS=1 BUILDTAGS="$(BUILDTAGS)" $(MAKE) test-integration-local'
# Intended for CI, shortcut 'build-container' since already running inside container.
test-integration-local:
hack/make.sh test-integration
$(CONTAINER_RUN) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; SKOPEO_CONTAINER_TESTS=1 BUILDTAGS="$(BUILDTAGS)" hack/make.sh test-integration'
# complicated set of options needed to run podman-in-podman
test-system: build-container
@@ -194,35 +162,24 @@ test-system: build-container
$(CONTAINER_CMD) --privileged \
-v $$DTEMP:/var/lib/containers:Z -v /run/systemd/journal/socket:/run/systemd/journal/socket \
"$(IMAGE)" \
bash -c 'BUILDTAGS="$(BUILDTAGS)" $(MAKE) test-system-local'; \
bash -c 'BUILDTAGS="$(BUILDTAGS)" hack/make.sh test-system'; \
rc=$$?; \
$(RM) -rf $$DTEMP; \
exit $$rc
# Intended for CI, shortcut 'build-container' since already running inside container.
test-system-local:
hack/make.sh test-system
test-unit: build-container
# Just call (make test unit-local) here instead of worrying about environment differences
$(CONTAINER_RUN) make test-unit-local BUILDTAGS='$(BUILDTAGS)'
validate: build-container
$(CONTAINER_RUN) make validate-local
$(CONTAINER_RUN) hack/make.sh validate-git-marks validate-gofmt validate-lint validate-vet
# This target is only intended for development, e.g. executing it from an IDE. Use (make test) for CI or pre-release testing.
test-all-local: validate-local validate-docs test-unit-local
test-all-local: validate-local test-unit-local
.PHONY: validate-local
validate-local:
hack/make.sh validate-git-marks validate-gofmt validate-lint validate-vet
# This invokes bin/skopeo, hence cannot be run as part of validate-local
.PHONY: validate-docs
validate-docs:
hack/man-page-checker
hack/xref-helpmsgs-manpages
test-unit-local:
$(GPGME_ENV) $(GO) test $(MOD_VENDOR) -tags "$(BUILDTAGS)" $$($(GO) list $(MOD_VENDOR) -tags "$(BUILDTAGS)" -e ./... | grep -v '^github\.com/containers/skopeo/\(integration\|vendor/.*\)$$')
@@ -232,4 +189,4 @@ vendor:
$(GO) mod verify
vendor-in-container:
podman run --privileged --rm --env HOME=/root -v `pwd`:/src -w /src docker.io/library/golang:1.16 make vendor
podman run --privileged --rm --env HOME=/root -v `pwd`:/src -w /src docker.io/library/golang:1.13 make vendor

View File

@@ -19,7 +19,6 @@ Skopeo works with API V2 container image registries such as [docker.io](https://
For example you can copy images from one registry to another, without requiring privilege.
* Inspecting a remote image showing its properties including its layers, without requiring you to pull the image to the host.
* Deleting an image from an image repository.
* Syncing an external image repository to an internal registry for air-gapped deployments.
* When required by the repository, skopeo can pass the appropriate credentials and certificates for authentication.
Skopeo operates on the following image and repository types:
@@ -122,7 +121,7 @@ $ skopeo inspect --config docker://registry.fedoraproject.org/fedora:latest | j
]
}
```
#### Show unverified image's digest
#### Show unverifed image's digest
```console
$ skopeo inspect docker://registry.fedoraproject.org/fedora:latest | jq '.Digest'
"sha256:655721ff613ee766a4126cb5e0d5ae81598e1b0c3bcf7017c36c4d72cb092fe9"
@@ -137,7 +136,7 @@ $ skopeo inspect docker://registry.fedoraproject.org/fedora:latest | jq '.Digest
* Container Storage backends
- [github.com/containers/storage](https://github.com/containers/storage) (Backend for [Podman](https://podman.io), [CRI-O](https://cri-o.io), [Buildah](https://buildah.io) and friends)
- github.com/containers/storage (Backend for [Podman](https://podman.io), [CRI-O](https://cri-o.io), [Buildah](https://buildah.io) and friends)
- Docker daemon storage
@@ -155,22 +154,17 @@ $ skopeo copy oci:busybox_ocilayout:latest dir:existingemptydirectory
$ skopeo delete docker://localhost:5000/imagename:latest
```
## Syncing registries
```console
$ skopeo sync --src docker --dest dir registry.example.com/busybox /media/usb
```
## Authenticating to a registry
#### Private registries with authentication
skopeo uses credentials from the --creds (for skopeo inspect|delete) or --src-creds|--dest-creds (for skopeo copy) flags, if set; otherwise it uses configuration set by skopeo login, podman login, buildah login, or docker login.
```console
$ skopeo login --username USER myregistrydomain.com:5000
$ skopeo login --user USER docker://myregistrydomain.com:5000
Password:
$ skopeo inspect docker://myregistrydomain.com:5000/busybox
{"Tag":"latest","Digest":"sha256:473bb2189d7b913ed7187a33d11e743fdc2f88931122a44d91a301b64419f092","RepoTags":["latest"],"Comment":"","Created":"2016-01-15T18:06:41.282540103Z","ContainerConfig":{"Hostname":"aded96b43f48","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":["/bin/sh","-c","#(nop) CMD [\"sh\"]"],"Image":"9e77fef7a1c9f989988c06620dabc4020c607885b959a2cbd7c2283c91da3e33","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"DockerVersion":"1.8.3","Author":"","Config":{"Hostname":"aded96b43f48","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":["sh"],"Image":"9e77fef7a1c9f989988c06620dabc4020c607885b959a2cbd7c2283c91da3e33","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"Architecture":"amd64","Os":"linux"}
$ skopeo logout myregistrydomain.com:5000
$ skopeo logout docker://myregistrydomain.com:5000
```
#### Using --creds directly
@@ -195,20 +189,6 @@ Contributing
Please read the [contribution guide](CONTRIBUTING.md) if you want to collaborate in the project.
## Commands
| Command | Description |
| -------------------------------------------------- | ---------------------------------------------------------------------------------------------|
| [skopeo-copy(1)](/docs/skopeo-copy.1.md) | Copy an image (manifest, filesystem layers, signatures) from one location to another. |
| [skopeo-delete(1)](/docs/skopeo-delete.1.md) | Mark the image-name for later deletion by the registry's garbage collector. |
| [skopeo-inspect(1)](/docs/skopeo-inspect.1.md) | Return low-level information about image-name in a registry. |
| [skopeo-list-tags(1)](/docs/skopeo-list-tags.1.md) | Return a list of tags for the transport-specific image repository. |
| [skopeo-login(1)](/docs/skopeo-login.1.md) | Login to a container registry. |
| [skopeo-logout(1)](/docs/skopeo-logout.1.md) | Logout of a container registry. |
| [skopeo-manifest-digest(1)](/docs/skopeo-manifest-digest.1.md) | Compute a manifest digest for a manifest-file and write it to standard output. |
| [skopeo-standalone-sign(1)](/docs/skopeo-standalone-sign.1.md) | Debugging tool - Publish and sign an image in one step. |
| [skopeo-standalone-verify(1)](/docs/skopeo-standalone-verify.1.md)| Verify an image signature. |
| [skopeo-sync(1)](/docs/skopeo-sync.1.md) | Synchronize images between container registries and local directories. |
License
-
skopeo is licensed under the Apache License, Version 2.0. See

View File

@@ -1,3 +1,3 @@
## Security and Disclosure Information Policy for the skopeo Project
The skopeo Project follows the [Security and Disclosure Information Policy](https://github.com/containers/common/blob/main/SECURITY.md) for the Containers Projects.
The skopeo Project follows the [Security and Disclosure Information Policy](https://github.com/containers/common/blob/master/SECURITY.md) for the Containers Projects.

View File

@@ -1,4 +1,3 @@
//go:build !containers_image_openpgp
// +build !containers_image_openpgp
package main

View File

@@ -4,49 +4,42 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"strings"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/copy"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/transports/alltransports"
"github.com/spf13/cobra"
encconfig "github.com/containers/ocicrypt/config"
enchelpers "github.com/containers/ocicrypt/helpers"
"github.com/spf13/cobra"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
)
type copyOptions struct {
global *globalOptions
deprecatedTLSVerify *deprecatedTLSVerifyOption
srcImage *imageOptions
destImage *imageDestOptions
retryOpts *retry.RetryOptions
additionalTags []string // For docker-archive: destinations, in addition to the name:tag specified as destination, also add these
removeSignatures bool // Do not copy signatures from the source image
signByFingerprint string // Sign the image using a GPG key with the specified fingerprint
digestFile string // Write digest to this file
format optionalString // Force conversion of the image to a specified format
quiet bool // Suppress output information when copying images
all bool // Copy all of the images if the source is a list
encryptLayer []int // The list of layers to encrypt
encryptionKeys []string // Keys needed to encrypt the image
decryptionKeys []string // Keys needed to decrypt the image
global *globalOptions
srcImage *imageOptions
destImage *imageDestOptions
additionalTags []string // For docker-archive: destinations, in addition to the name:tag specified as destination, also add these
removeSignatures bool // Do not copy signatures from the source image
signByFingerprint string // Sign the image using a GPG key with the specified fingerprint
format optionalString // Force conversion of the image to a specified format
quiet bool // Suppress output information when copying images
all bool // Copy all of the images if the source is a list
encryptLayer []int // The list of layers to encrypt
encryptionKeys []string // Keys needed to encrypt the image
decryptionKeys []string // Keys needed to decrypt the image
}
func copyCmd(global *globalOptions) *cobra.Command {
sharedFlags, sharedOpts := sharedImageFlags()
deprecatedTLSVerifyFlags, deprecatedTLSVerifyOpt := deprecatedTLSVerifyFlags()
srcFlags, srcOpts := imageFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "src-", "screds")
destFlags, destOpts := imageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
retryFlags, retryOpts := retryFlags()
srcFlags, srcOpts := imageFlags(global, sharedOpts, "src-", "screds")
destFlags, destOpts := imageDestFlags(global, sharedOpts, "dest-", "dcreds")
opts := copyOptions{global: global,
deprecatedTLSVerify: deprecatedTLSVerifyOpt,
srcImage: srcOpts,
destImage: destOpts,
retryOpts: retryOpts,
srcImage: srcOpts,
destImage: destOpts,
}
cmd := &cobra.Command{
Use: "copy [command options] SOURCE-IMAGE DESTINATION-IMAGE",
@@ -59,22 +52,19 @@ Supported transports:
See skopeo(1) section "IMAGE NAMES" for the expected format
`, strings.Join(transports.ListNames(), ", ")),
RunE: commandAction(opts.run),
Example: `skopeo copy docker://quay.io/skopeo/stable:latest docker://registry.example.com/skopeo:latest`,
Example: `skopeo copy --sign-by dev@example.com container-storage:example/busybox:streaming docker://example/busybox:gold`,
}
adjustUsage(cmd)
flags := cmd.Flags()
flags.AddFlagSet(&sharedFlags)
flags.AddFlagSet(&deprecatedTLSVerifyFlags)
flags.AddFlagSet(&srcFlags)
flags.AddFlagSet(&destFlags)
flags.AddFlagSet(&retryFlags)
flags.StringSliceVar(&opts.additionalTags, "additional-tag", []string{}, "additional tags (supports docker-archive)")
flags.BoolVarP(&opts.quiet, "quiet", "q", false, "Suppress output information when copying images")
flags.BoolVarP(&opts.all, "all", "a", false, "Copy all images if SOURCE-IMAGE is a list")
flags.BoolVar(&opts.removeSignatures, "remove-signatures", false, "Do not copy signatures from SOURCE-IMAGE")
flags.StringVar(&opts.signByFingerprint, "sign-by", "", "Sign the image using a GPG key with the specified `FINGERPRINT`")
flags.StringVar(&opts.digestFile, "digestfile", "", "Write the digest of the pushed image to the specified file")
flags.VarP(newOptionalStringValue(&opts.format), "format", "f", `MANIFEST TYPE (oci, v2s1, or v2s2) to use in the destination (default is manifest type of source, with fallbacks)`)
flags.VarP(newOptionalStringValue(&opts.format), "format", "f", `MANIFEST TYPE (oci, v2s1, or v2s2) to use when saving image to directory using the 'dir:' transport (default is manifest type of source)`)
flags.StringSliceVar(&opts.encryptionKeys, "encryption-key", []string{}, "*Experimental* key with the encryption protocol to use needed to encrypt the image (e.g. jwe:/path/to/key.pem)")
flags.IntSliceVar(&opts.encryptLayer, "encrypt-layer", []int{}, "*Experimental* the 0-indexed layer indices, with support for negative indexing (e.g. 0 is the first layer, -1 is the last layer)")
flags.StringSliceVar(&opts.decryptionKeys, "decryption-key", []string{}, "*Experimental* key needed to decrypt the image")
@@ -85,7 +75,6 @@ func (opts *copyOptions) run(args []string, stdout io.Writer) error {
if len(args) != 2 {
return errorShouldDisplayUsage{errors.New("Exactly two arguments expected")}
}
opts.deprecatedTLSVerify.warnIfUsed([]string{"--src-tls-verify", "--dest-tls-verify"})
imageNames := args
if err := reexecIfNecessaryForImages(imageNames...); err != nil {
@@ -118,9 +107,15 @@ func (opts *copyOptions) run(args []string, stdout io.Writer) error {
var manifestType string
if opts.format.present {
manifestType, err = parseManifestFormat(opts.format.value)
if err != nil {
return err
switch opts.format.value {
case "oci":
manifestType = imgspecv1.MediaTypeImageManifest
case "v2s1":
manifestType = manifest.DockerV2Schema1SignedMediaType
case "v2s2":
manifestType = manifest.DockerV2Schema2MediaType
default:
return fmt.Errorf("unknown format %q. Choose one of the supported formats: 'oci', 'v2s1', or 'v2s2'", opts.format.value)
}
}
@@ -183,31 +178,17 @@ func (opts *copyOptions) run(args []string, stdout io.Writer) error {
decConfig = cc.DecryptConfig
}
return retry.RetryIfNecessary(ctx, func() error {
manifestBytes, err := copy.Image(ctx, policyContext, destRef, srcRef, &copy.Options{
RemoveSignatures: opts.removeSignatures,
SignBy: opts.signByFingerprint,
ReportWriter: stdout,
SourceCtx: sourceCtx,
DestinationCtx: destinationCtx,
ForceManifestMIMEType: manifestType,
ImageListSelection: imageListSelection,
OciDecryptConfig: decConfig,
OciEncryptLayers: encLayers,
OciEncryptConfig: encConfig,
})
if err != nil {
return err
}
if opts.digestFile != "" {
manifestDigest, err := manifest.Digest(manifestBytes)
if err != nil {
return err
}
if err = ioutil.WriteFile(opts.digestFile, []byte(manifestDigest.String()), 0644); err != nil {
return fmt.Errorf("Failed to write digest to file %q: %w", opts.digestFile, err)
}
}
return nil
}, opts.retryOpts)
_, err = copy.Image(ctx, policyContext, destRef, srcRef, &copy.Options{
RemoveSignatures: opts.removeSignatures,
SignBy: opts.signByFingerprint,
ReportWriter: stdout,
SourceCtx: sourceCtx,
DestinationCtx: destinationCtx,
ForceManifestMIMEType: manifestType,
ImageListSelection: imageListSelection,
OciDecryptConfig: decConfig,
OciEncryptLayers: encLayers,
OciEncryptConfig: encConfig,
})
return err
}

View File

@@ -6,26 +6,22 @@ import (
"io"
"strings"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/transports/alltransports"
"github.com/spf13/cobra"
)
type deleteOptions struct {
global *globalOptions
image *imageOptions
retryOpts *retry.RetryOptions
global *globalOptions
image *imageOptions
}
func deleteCmd(global *globalOptions) *cobra.Command {
sharedFlags, sharedOpts := sharedImageFlags()
imageFlags, imageOpts := imageFlags(global, sharedOpts, nil, "", "")
retryFlags, retryOpts := retryFlags()
imageFlags, imageOpts := imageFlags(global, sharedOpts, "", "")
opts := deleteOptions{
global: global,
image: imageOpts,
retryOpts: retryOpts,
global: global,
image: imageOpts,
}
cmd := &cobra.Command{
Use: "delete [command options] IMAGE-NAME",
@@ -42,7 +38,6 @@ See skopeo(1) section "IMAGE NAMES" for the expected format
flags := cmd.Flags()
flags.AddFlagSet(&sharedFlags)
flags.AddFlagSet(&imageFlags)
flags.AddFlagSet(&retryFlags)
return cmd
}
@@ -68,8 +63,5 @@ func (opts *deleteOptions) run(args []string, stdout io.Writer) error {
ctx, cancel := opts.global.commandTimeoutContext()
defer cancel()
return retry.RetryIfNecessary(ctx, func() error {
return ref.DeleteImage(ctx, sys)
}, opts.retryOpts)
return ref.DeleteImage(ctx, sys)
}

View File

@@ -4,42 +4,47 @@ import (
"encoding/json"
"fmt"
"io"
"os"
"strings"
"text/tabwriter"
"text/template"
"time"
"github.com/containers/common/pkg/report"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/docker"
"github.com/containers/image/v5/image"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
"github.com/containers/skopeo/cmd/skopeo/inspect"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
// inspectOutput is the output format of (skopeo inspect), primarily so that we can format it with a simple json.MarshalIndent.
type inspectOutput struct {
Name string `json:",omitempty"`
Tag string `json:",omitempty"`
Digest digest.Digest
RepoTags []string
Created *time.Time
DockerVersion string
Labels map[string]string
Architecture string
Os string
Layers []string
Env []string
}
type inspectOptions struct {
global *globalOptions
image *imageOptions
retryOpts *retry.RetryOptions
format string
raw bool // Output the raw manifest instead of parsing information about the image
config bool // Output the raw config blob instead of parsing information about the image
global *globalOptions
image *imageOptions
raw bool // Output the raw manifest instead of parsing information about the image
config bool // Output the raw config blob instead of parsing information about the image
}
func inspectCmd(global *globalOptions) *cobra.Command {
sharedFlags, sharedOpts := sharedImageFlags()
imageFlags, imageOpts := imageFlags(global, sharedOpts, nil, "", "")
retryFlags, retryOpts := retryFlags()
imageFlags, imageOpts := imageFlags(global, sharedOpts, "", "")
opts := inspectOptions{
global: global,
image: imageOpts,
retryOpts: retryOpts,
global: global,
image: imageOpts,
}
cmd := &cobra.Command{
Use: "inspect [command options] IMAGE-NAME",
@@ -50,38 +55,25 @@ Supported transports:
See skopeo(1) section "IMAGE NAMES" for the expected format
`, strings.Join(transports.ListNames(), ", ")),
RunE: commandAction(opts.run),
Example: `skopeo inspect docker://registry.fedoraproject.org/fedora
skopeo inspect --config docker://docker.io/alpine
skopeo inspect --format "Name: {{.Name}} Digest: {{.Digest}}" docker://registry.access.redhat.com/ubi8`,
RunE: commandAction(opts.run),
Example: `skopeo inspect docker://docker.io/fedora`,
}
adjustUsage(cmd)
flags := cmd.Flags()
flags.BoolVar(&opts.raw, "raw", false, "output raw manifest or configuration")
flags.BoolVar(&opts.config, "config", false, "output configuration")
flags.StringVarP(&opts.format, "format", "f", "", "Format the output to a Go template")
flags.AddFlagSet(&sharedFlags)
flags.AddFlagSet(&imageFlags)
flags.AddFlagSet(&retryFlags)
return cmd
}
func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error) {
var (
rawManifest []byte
src types.ImageSource
imgInspect *types.ImageInspectInfo
data []interface{}
)
ctx, cancel := opts.global.commandTimeoutContext()
defer cancel()
if len(args) != 1 {
return errors.New("Exactly one argument expected")
}
if opts.raw && opts.format != "" {
return errors.New("raw output does not support format option")
}
imageName := args[0]
if err := reexecIfNecessaryForImages(imageName); err != nil {
@@ -93,11 +85,9 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
return err
}
if err := retry.RetryIfNecessary(ctx, func() error {
src, err = parseImageSource(ctx, opts.image, imageName)
return err
}, opts.retryOpts); err != nil {
return errors.Wrapf(err, "Error parsing image name %q", imageName)
src, err := parseImageSource(ctx, opts.image, imageName)
if err != nil {
return fmt.Errorf("Error parsing image name %q: %v", imageName, err)
}
defer func() {
@@ -106,11 +96,9 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
}
}()
if err := retry.RetryIfNecessary(ctx, func() error {
rawManifest, _, err = src.GetManifest(ctx, nil)
return err
}, opts.retryOpts); err != nil {
return errors.Wrapf(err, "Error retrieving manifest for image")
rawManifest, _, err := src.GetManifest(ctx, nil)
if err != nil {
return fmt.Errorf("Error retrieving manifest for image: %v", err)
}
if opts.raw && !opts.config {
@@ -118,65 +106,45 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
if err != nil {
return fmt.Errorf("Error writing manifest to standard output: %v", err)
}
return nil
}
img, err := image.FromUnparsedImage(ctx, sys, image.UnparsedInstance(src, nil))
if err != nil {
return errors.Wrapf(err, "Error parsing manifest for image")
return fmt.Errorf("Error parsing manifest for image: %v", err)
}
if opts.config && opts.raw {
var configBlob []byte
if err := retry.RetryIfNecessary(ctx, func() error {
configBlob, err = img.ConfigBlob(ctx)
return err
}, opts.retryOpts); err != nil {
return errors.Wrapf(err, "Error reading configuration blob")
configBlob, err := img.ConfigBlob(ctx)
if err != nil {
return fmt.Errorf("Error reading configuration blob: %v", err)
}
_, err = stdout.Write(configBlob)
if err != nil {
return errors.Wrapf(err, "Error writing configuration blob to standard output")
return fmt.Errorf("Error writing configuration blob to standard output: %v", err)
}
return nil
} else if opts.config {
var config *v1.Image
if err := retry.RetryIfNecessary(ctx, func() error {
config, err = img.OCIConfig(ctx)
return err
}, opts.retryOpts); err != nil {
return errors.Wrapf(err, "Error reading OCI-formatted configuration data")
}
if report.IsJSON(opts.format) || opts.format == "" {
var out []byte
out, err = json.MarshalIndent(config, "", " ")
if err == nil {
fmt.Fprintf(stdout, "%s\n", string(out))
}
} else {
row := "{{range . }}" + report.NormalizeFormat(opts.format) + "{{end}}"
data = append(data, config)
err = printTmpl(row, data)
}
config, err := img.OCIConfig(ctx)
if err != nil {
return errors.Wrapf(err, "Error writing OCI-formatted configuration data to standard output")
return fmt.Errorf("Error reading OCI-formatted configuration data: %v", err)
}
err = json.NewEncoder(stdout).Encode(config)
if err != nil {
return fmt.Errorf("Error writing OCI-formatted configuration data to standard output: %v", err)
}
return nil
}
if err := retry.RetryIfNecessary(ctx, func() error {
imgInspect, err = img.Inspect(ctx)
return err
}, opts.retryOpts); err != nil {
imgInspect, err := img.Inspect(ctx)
if err != nil {
return err
}
outputData := inspect.Output{
outputData := inspectOutput{
Name: "", // Set below if DockerReference() is known
Tag: imgInspect.Tag,
// Digest is set below.
RepoTags: []string{}, // Possibly overridden for docker.Transport.
RepoTags: []string{}, // Possibly overriden for docker.Transport.
Created: imgInspect.Created,
DockerVersion: imgInspect.DockerVersion,
Labels: imgInspect.Labels,
@@ -187,7 +155,7 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
}
outputData.Digest, err = manifest.Digest(rawManifest)
if err != nil {
return errors.Wrapf(err, "Error computing manifest digest")
return fmt.Errorf("Error computing manifest digest: %v", err)
}
if dockerRef := img.Reference().DockerReference(); dockerRef != nil {
outputData.Name = dockerRef.Name()
@@ -205,28 +173,15 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
// In addition, AWS ECR rejects it with 403 (Forbidden) if the "ecr:ListImages"
// action is not allowed.
if !strings.Contains(err.Error(), "401") && !strings.Contains(err.Error(), "403") {
return errors.Wrapf(err, "Error determining repository tags")
return fmt.Errorf("Error determining repository tags: %v", err)
}
logrus.Warnf("Registry disallows tag list retrieval; skipping")
}
}
if report.IsJSON(opts.format) || opts.format == "" {
out, err := json.MarshalIndent(outputData, "", " ")
if err == nil {
fmt.Fprintf(stdout, "%s\n", string(out))
}
return err
}
row := "{{range . }}" + report.NormalizeFormat(opts.format) + "{{end}}"
data = append(data, outputData)
return printTmpl(row, data)
}
func printTmpl(row string, data []interface{}) error {
t, err := template.New("skopeo inspect").Parse(row)
out, err := json.MarshalIndent(outputData, "", " ")
if err != nil {
return err
}
w := tabwriter.NewWriter(os.Stdout, 8, 2, 2, ' ', 0)
return t.Execute(w, data)
fmt.Fprintf(stdout, "%s\n", string(out))
return nil
}

View File

@@ -1,23 +0,0 @@
package inspect
import (
"time"
digest "github.com/opencontainers/go-digest"
)
// Output is the output format of (skopeo inspect),
// primarily so that we can format it with a simple json.MarshalIndent.
type Output struct {
Name string `json:",omitempty"`
Tag string `json:",omitempty"`
Digest digest.Digest
RepoTags []string
Created *time.Time
DockerVersion string
Labels map[string]string
Architecture string
Os string
Layers []string
Env []string
}

View File

@@ -7,7 +7,6 @@ import (
"os"
"strings"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/directory"
"github.com/containers/image/v5/image"
"github.com/containers/image/v5/pkg/blobinfocache"
@@ -18,19 +17,16 @@ import (
)
type layersOptions struct {
global *globalOptions
image *imageOptions
retryOpts *retry.RetryOptions
global *globalOptions
image *imageOptions
}
func layersCmd(global *globalOptions) *cobra.Command {
sharedFlags, sharedOpts := sharedImageFlags()
imageFlags, imageOpts := imageFlags(global, sharedOpts, nil, "", "")
retryFlags, retryOpts := retryFlags()
imageFlags, imageOpts := imageFlags(global, sharedOpts, "", "")
opts := layersOptions{
global: global,
image: imageOpts,
retryOpts: retryOpts,
global: global,
image: imageOpts,
}
cmd := &cobra.Command{
Hidden: true,
@@ -42,7 +38,6 @@ func layersCmd(global *globalOptions) *cobra.Command {
flags := cmd.Flags()
flags.AddFlagSet(&sharedFlags)
flags.AddFlagSet(&imageFlags)
flags.AddFlagSet(&retryFlags)
return cmd
}
@@ -65,20 +60,12 @@ func (opts *layersOptions) run(args []string, stdout io.Writer) (retErr error) {
return err
}
cache := blobinfocache.DefaultCache(sys)
var (
rawSource types.ImageSource
src types.ImageCloser
)
if err = retry.RetryIfNecessary(ctx, func() error {
rawSource, err = parseImageSource(ctx, opts.image, imageName)
return err
}, opts.retryOpts); err != nil {
rawSource, err := parseImageSource(ctx, opts.image, imageName)
if err != nil {
return err
}
if err = retry.RetryIfNecessary(ctx, func() error {
src, err = image.FromSource(ctx, sys, rawSource)
return err
}, opts.retryOpts); err != nil {
src, err := image.FromSource(ctx, sys, rawSource)
if err != nil {
if closeErr := rawSource.Close(); closeErr != nil {
return errors.Wrapf(err, " (close error: %v)", closeErr)
}
@@ -142,14 +129,8 @@ func (opts *layersOptions) run(args []string, stdout io.Writer) (retErr error) {
}()
for _, bd := range blobDigests {
var (
r io.ReadCloser
blobSize int64
)
if err = retry.RetryIfNecessary(ctx, func() error {
r, blobSize, err = rawSource.GetBlob(ctx, types.BlobInfo{Digest: bd.digest, Size: -1}, cache)
return err
}, opts.retryOpts); err != nil {
r, blobSize, err := rawSource.GetBlob(ctx, types.BlobInfo{Digest: bd.digest, Size: -1}, cache)
if err != nil {
return err
}
if _, err := dest.PutBlob(ctx, r, types.BlobInfo{Digest: bd.digest, Size: blobSize}, cache, bd.isConfig); err != nil {
@@ -160,11 +141,8 @@ func (opts *layersOptions) run(args []string, stdout io.Writer) (retErr error) {
}
}
var manifest []byte
if err = retry.RetryIfNecessary(ctx, func() error {
manifest, _, err = src.Manifest(ctx)
return err
}, opts.retryOpts); err != nil {
manifest, _, err := src.Manifest(ctx)
if err != nil {
return err
}
if err := dest.PutManifest(ctx, manifest, nil); err != nil {

View File

@@ -7,7 +7,6 @@ import (
"io"
"strings"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/docker"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/transports/alltransports"
@@ -23,20 +22,17 @@ type tagListOutput struct {
}
type tagsOptions struct {
global *globalOptions
image *imageOptions
retryOpts *retry.RetryOptions
global *globalOptions
image *imageOptions
}
func tagsCmd(global *globalOptions) *cobra.Command {
sharedFlags, sharedOpts := sharedImageFlags()
imageFlags, imageOpts := dockerImageFlags(global, sharedOpts, nil, "", "")
retryFlags, retryOpts := retryFlags()
imageFlags, imageOpts := dockerImageFlags(global, sharedOpts, "", "")
opts := tagsOptions{
global: global,
image: imageOpts,
retryOpts: retryOpts,
global: global,
image: imageOpts,
}
cmd := &cobra.Command{
Use: "list-tags [command options] REPOSITORY-NAME",
@@ -55,7 +51,6 @@ See skopeo-list-tags(1) section "REPOSITORY NAMES" for the expected format
flags := cmd.Flags()
flags.AddFlagSet(&sharedFlags)
flags.AddFlagSet(&imageFlags)
flags.AddFlagSet(&retryFlags)
return cmd
}
@@ -123,12 +118,8 @@ func (opts *tagsOptions) run(args []string, stdout io.Writer) (retErr error) {
return err
}
var repositoryName string
var tagListing []string
if err = retry.RetryIfNecessary(ctx, func() error {
repositoryName, tagListing, err = listDockerTags(ctx, sys, imgRef)
return err
}, opts.retryOpts); err != nil {
repositoryName, tagListing, err := listDockerTags(ctx, sys, imgRef)
if err != nil {
return err
}

View File

@@ -5,7 +5,6 @@ import (
"github.com/containers/image/v5/transports/alltransports"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Tests the kinds of inputs allowed and expected to the command
@@ -18,10 +17,10 @@ func TestDockerRepositoryReferenceParser(t *testing.T) {
} {
ref, err := parseDockerRepositoryReference(test[0])
require.NoError(t, err)
expected, err := alltransports.ParseImageName(test[0])
require.NoError(t, err)
assert.Equal(t, expected.DockerReference().Name(), ref.DockerReference().Name(), "Mismatched parse result for input %v", test[0])
if assert.NoError(t, err, "Could not parse, got error on %v", test[0]) {
assert.Equal(t, expected.DockerReference().Name(), ref.DockerReference().Name(), "Mismatched parse result for input %v", test[0])
}
}
for _, test := range [][]string{

View File

@@ -12,6 +12,7 @@ import (
type loginOptions struct {
global *globalOptions
loginOpts auth.LoginOptions
getLogin optionalBool
tlsVerify optionalBool
}
@@ -20,7 +21,7 @@ func loginCmd(global *globalOptions) *cobra.Command {
global: global,
}
cmd := &cobra.Command{
Use: "login [command options] REGISTRY",
Use: "login",
Short: "Login to a container registry",
Long: "Login to a container registry on a specified server.",
RunE: commandAction(opts.run),
@@ -38,7 +39,6 @@ func (opts *loginOptions) run(args []string, stdout io.Writer) error {
defer cancel()
opts.loginOpts.Stdout = stdout
opts.loginOpts.Stdin = os.Stdin
opts.loginOpts.AcceptRepositories = true
sys := opts.global.newSystemContext()
if opts.tlsVerify.present {
sys.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!opts.tlsVerify.value)

View File

@@ -4,14 +4,12 @@ import (
"io"
"github.com/containers/common/pkg/auth"
"github.com/containers/image/v5/types"
"github.com/spf13/cobra"
)
type logoutOptions struct {
global *globalOptions
logoutOpts auth.LogoutOptions
tlsVerify optionalBool
}
func logoutCmd(global *globalOptions) *cobra.Command {
@@ -19,25 +17,19 @@ func logoutCmd(global *globalOptions) *cobra.Command {
global: global,
}
cmd := &cobra.Command{
Use: "logout [command options] REGISTRY",
Use: "logout",
Short: "Logout of a container registry",
Long: "Logout of a container registry on a specified server.",
RunE: commandAction(opts.run),
Example: `skopeo logout quay.io`,
}
adjustUsage(cmd)
flags := cmd.Flags()
optionalBoolFlag(flags, &opts.tlsVerify, "tls-verify", "require HTTPS and verify certificates when accessing the registry")
flags.AddFlagSet(auth.GetLogoutFlags(&opts.logoutOpts))
cmd.Flags().AddFlagSet(auth.GetLogoutFlags(&opts.logoutOpts))
return cmd
}
func (opts *logoutOptions) run(args []string, stdout io.Writer) error {
opts.logoutOpts.Stdout = stdout
opts.logoutOpts.AcceptRepositories = true
sys := opts.global.newSystemContext()
if opts.tlsVerify.present {
sys.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!opts.tlsVerify.value)
}
return auth.Logout(sys, &opts.logoutOpts, args)
}

View File

@@ -17,8 +17,6 @@ import (
// and will be populated by the Makefile
var gitCommit = ""
var defaultUserAgent = "skopeo/" + version.Version
type globalOptions struct {
debug bool // Enable debug output
tlsVerify optionalBool // Require HTTPS and verify certificates (for docker: and docker-daemon:)
@@ -45,17 +43,6 @@ func createApp() (*cobra.Command, *globalOptions) {
},
SilenceUsage: true,
SilenceErrors: true,
// Currently, skopeo uses manually written completions. Cobra allows
// for auto-generating completions for various shells. Podman is
// already making us of that. If Skopeo decides to follow, please
// remove the line below (and hide the `completion` command).
CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true},
// This is documented to parse "local" (non-PersistentFlags) flags of parent commands before
// running subcommands and handling their options. We don't really run into such cases,
// because all of our flags on rootCommand are in PersistentFlags, except for the deprecated --tls-verify;
// in that case we need TraverseChildren so that we can distinguish between
// (skopeo --tls-verify inspect) (causes a warning) and (skopeo inspect --tls-verify) (no warning).
TraverseChildren: true,
}
if gitCommit != "" {
rootCommand.Version = fmt.Sprintf("%s commit: %s", version.Version, gitCommit)
@@ -66,6 +53,8 @@ func createApp() (*cobra.Command, *globalOptions) {
var dummyVersion bool
rootCommand.Flags().BoolVarP(&dummyVersion, "version", "v", false, "Version for Skopeo")
rootCommand.PersistentFlags().BoolVar(&opts.debug, "debug", false, "enable debug output")
flag := optionalBoolFlag(rootCommand.PersistentFlags(), &opts.tlsVerify, "tls-verify", "Require HTTPS and verify certificates when accessing the registry")
flag.Hidden = true
rootCommand.PersistentFlags().StringVar(&opts.policyPath, "policy", "", "Path to a trust policy file")
rootCommand.PersistentFlags().BoolVar(&opts.insecurePolicy, "insecure-policy", false, "run the tool without any policy check")
rootCommand.PersistentFlags().StringVar(&opts.registriesDirPath, "registries.d", "", "use registry configuration files in `DIR` (e.g. for container signature storage)")
@@ -78,8 +67,6 @@ func createApp() (*cobra.Command, *globalOptions) {
logrus.Fatal("unable to mark registries-conf flag as hidden")
}
rootCommand.PersistentFlags().StringVar(&opts.tmpDir, "tmpdir", "", "directory used to store temporary files")
flag := optionalBoolFlag(rootCommand.Flags(), &opts.tlsVerify, "tls-verify", "Require HTTPS and verify certificates when accessing the registry")
flag.Hidden = true
rootCommand.AddCommand(
copyCmd(&opts),
deleteCmd(&opts),
@@ -156,7 +143,6 @@ func (opts *globalOptions) newSystemContext() *types.SystemContext {
VariantChoice: opts.overrideVariant,
SystemRegistriesConfPath: opts.registriesConfPath,
BigFilesTemporaryDir: opts.tmpDir,
DockerRegistryUserAgent: defaultUserAgent,
}
// DEPRECATED: We support this for backward compatibility, but override it if a per-image flag is provided.
if opts.tlsVerify.present {

View File

@@ -23,10 +23,7 @@ func TestGlobalOptionsNewSystemContext(t *testing.T) {
// Default state
opts, _ := fakeGlobalOptions(t, []string{})
res := opts.newSystemContext()
assert.Equal(t, &types.SystemContext{
// User-Agent is set by default.
DockerRegistryUserAgent: defaultUserAgent,
}, res)
assert.Equal(t, &types.SystemContext{}, res)
// Set everything to non-default values.
opts, _ = fakeGlobalOptions(t, []string{
"--registries.d", "/srv/registries.d",
@@ -46,6 +43,5 @@ func TestGlobalOptionsNewSystemContext(t *testing.T) {
BigFilesTemporaryDir: "/srv",
SystemRegistriesConfPath: "/srv/registries.conf",
DockerInsecureSkipTLSVerify: types.OptionalBoolTrue,
DockerRegistryUserAgent: defaultUserAgent,
}, res)
}

View File

@@ -16,7 +16,7 @@ type manifestDigestOptions struct {
func manifestDigestCmd() *cobra.Command {
var opts manifestDigestOptions
cmd := &cobra.Command{
Use: "manifest-digest MANIFEST-FILE",
Use: "manifest-digest MANIFEST",
Short: "Compute a manifest digest of a file",
RunE: commandAction(opts.run),
Example: "skopeo manifest-digest manifest.json",

View File

@@ -17,8 +17,8 @@ func TestManifestDigest(t *testing.T) {
}
// Error reading manifest
out, err := runSkopeo("manifest-digest", "/this/does/not/exist")
assertTestFailed(t, out, err, "/this/does/not/exist")
out, err := runSkopeo("manifest-digest", "/this/doesnt/exist")
assertTestFailed(t, out, err, "/this/doesnt/exist")
// Error computing manifest
out, err = runSkopeo("manifest-digest", "fixtures/v2s1-invalid-signatures.manifest.json")

View File

@@ -18,7 +18,7 @@ type standaloneSignOptions struct {
func standaloneSignCmd() *cobra.Command {
opts := standaloneSignOptions{}
cmd := &cobra.Command{
Use: "standalone-sign [command options] MANIFEST DOCKER-REFERENCE KEY-FINGERPRINT --output|-o SIGNATURE",
Use: "standalone-sign [command options] MANIFEST DOCKER-REFERENCE KEY-FINGERPRINT",
Short: "Create a signature using local files",
RunE: commandAction(opts.run),
}

View File

@@ -58,8 +58,8 @@ func TestStandaloneSign(t *testing.T) {
// Error reading manifest
out, err := runSkopeo("standalone-sign", "-o", "/dev/null",
"/this/does/not/exist", dockerReference, fixturesTestKeyFingerprint)
assertTestFailed(t, out, err, "/this/does/not/exist")
"/this/doesnt/exist", dockerReference, fixturesTestKeyFingerprint)
assertTestFailed(t, out, err, "/this/doesnt/exist")
// Invalid Docker reference
out, err = runSkopeo("standalone-sign", "-o", "/dev/null",
@@ -117,14 +117,14 @@ func TestStandaloneVerify(t *testing.T) {
}
// Error reading manifest
out, err := runSkopeo("standalone-verify", "/this/does/not/exist",
out, err := runSkopeo("standalone-verify", "/this/doesnt/exist",
dockerReference, fixturesTestKeyFingerprint, signaturePath)
assertTestFailed(t, out, err, "/this/does/not/exist")
assertTestFailed(t, out, err, "/this/doesnt/exist")
// Error reading signature
out, err = runSkopeo("standalone-verify", manifestPath,
dockerReference, fixturesTestKeyFingerprint, "/this/does/not/exist")
assertTestFailed(t, out, err, "/this/does/not/exist")
dockerReference, fixturesTestKeyFingerprint, "/this/doesnt/exist")
assertTestFailed(t, out, err, "/this/doesnt/exist")
// Error verifying signature
out, err = runSkopeo("standalone-verify", manifestPath,
@@ -151,8 +151,8 @@ func TestUntrustedSignatureDump(t *testing.T) {
// Error reading manifest
out, err := runSkopeo("untrusted-signature-dump-without-verification",
"/this/does/not/exist")
assertTestFailed(t, out, err, "/this/does/not/exist")
"/this/doesnt/exist")
assertTestFailed(t, out, err, "/this/doesnt/exist")
// Error reading signature (input is not a signature)
out, err = runSkopeo("untrusted-signature-dump-without-verification", "fixtures/image.manifest.json")

View File

@@ -11,14 +11,12 @@ import (
"regexp"
"strings"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/copy"
"github.com/containers/image/v5/directory"
"github.com/containers/image/v5/docker"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@@ -27,28 +25,24 @@ import (
// syncOptions contains information retrieved from the skopeo sync command line.
type syncOptions struct {
global *globalOptions // Global (not command dependent) skopeo options
deprecatedTLSVerify *deprecatedTLSVerifyOption
srcImage *imageOptions // Source image options
destImage *imageDestOptions // Destination image options
retryOpts *retry.RetryOptions
removeSignatures bool // Do not copy signatures from the source image
signByFingerprint string // Sign the image using a GPG key with the specified fingerprint
format optionalString // Force conversion of the image to a specified format
source string // Source repository name
destination string // Destination registry name
scoped bool // When true, namespace copied images at destination using the source repository name
all bool // Copy all of the images if an image in the source is a list
global *globalOptions // Global (not command dependant) skopeo options
srcImage *imageOptions // Source image options
destImage *imageDestOptions // Destination image options
removeSignatures bool // Do not copy signatures from the source image
signByFingerprint string // Sign the image using a GPG key with the specified fingerprint
source string // Source repository name
destination string // Destination registry name
scoped bool // When true, namespace copied images at destination using the source repository name
}
// repoDescriptor contains information of a single repository used as a sync source.
type repoDescriptor struct {
DirBasePath string // base path when source is 'dir'
ImageRefs []types.ImageReference // List of tagged image found for the repository
Context *types.SystemContext // SystemContext for the sync command
DirBasePath string // base path when source is 'dir'
TaggedImages []types.ImageReference // List of tagged image found for the repository
Context *types.SystemContext // SystemContext for the sync command
}
// tlsVerifyConfig is an implementation of the Unmarshaler interface, used to
// tlsVerify is an implementation of the Unmarshaler interface, used to
// customize the unmarshaling behaviour of the tls-verify YAML key.
type tlsVerifyConfig struct {
skip types.OptionalBool // skip TLS verification check (false by default)
@@ -57,7 +51,7 @@ type tlsVerifyConfig struct {
// registrySyncConfig contains information about a single registry, read from
// the source YAML file
type registrySyncConfig struct {
Images map[string][]string // Images map images name to slices with the images' references (tags, digests)
Images map[string][]string // Images map images name to slices with the images' tags
ImagesByTagRegex map[string]string `yaml:"images-by-tag-regex"` // Images map images name to regular expression with the images' tags
Credentials types.DockerAuthConfig // Username and password used to authenticate with the registry
TLSVerify tlsVerifyConfig `yaml:"tls-verify"` // TLS verification mode (enabled by default)
@@ -69,29 +63,25 @@ type sourceConfig map[string]registrySyncConfig
func syncCmd(global *globalOptions) *cobra.Command {
sharedFlags, sharedOpts := sharedImageFlags()
deprecatedTLSVerifyFlags, deprecatedTLSVerifyOpt := deprecatedTLSVerifyFlags()
srcFlags, srcOpts := dockerImageFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "src-", "screds")
destFlags, destOpts := dockerImageFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
retryFlags, retryOpts := retryFlags()
srcFlags, srcOpts := dockerImageFlags(global, sharedOpts, "src-", "screds")
destFlags, destOpts := dockerImageFlags(global, sharedOpts, "dest-", "dcreds")
opts := syncOptions{
global: global,
deprecatedTLSVerify: deprecatedTLSVerifyOpt,
srcImage: srcOpts,
destImage: &imageDestOptions{imageOptions: destOpts},
retryOpts: retryOpts,
global: global,
srcImage: srcOpts,
destImage: &imageDestOptions{imageOptions: destOpts},
}
cmd := &cobra.Command{
Use: "sync [command options] --src TRANSPORT --dest TRANSPORT SOURCE DESTINATION",
Use: "sync [command options] --src SOURCE-LOCATION --dest DESTINATION-LOCATION SOURCE DESTINATION",
Short: "Synchronize one or more images from one location to another",
Long: `Copy all the images from a SOURCE to a DESTINATION.
Long: fmt.Sprint(`Copy all the images from a SOURCE to a DESTINATION.
Allowed SOURCE transports (specified with --src): docker, dir, yaml.
Allowed DESTINATION transports (specified with --dest): docker, dir.
See skopeo-sync(1) for details.
`,
`),
RunE: commandAction(opts.run),
Example: `skopeo sync --src docker --dest dir --scoped registry.example.com/busybox /media/usb`,
}
@@ -99,20 +89,16 @@ See skopeo-sync(1) for details.
flags := cmd.Flags()
flags.BoolVar(&opts.removeSignatures, "remove-signatures", false, "Do not copy signatures from SOURCE images")
flags.StringVar(&opts.signByFingerprint, "sign-by", "", "Sign the image using a GPG key with the specified `FINGERPRINT`")
flags.VarP(newOptionalStringValue(&opts.format), "format", "f", `MANIFEST TYPE (oci, v2s1, or v2s2) to use when syncing image(s) to a destination (default is manifest type of source, with fallbacks)`)
flags.StringVarP(&opts.source, "src", "s", "", "SOURCE transport type")
flags.StringVarP(&opts.destination, "dest", "d", "", "DESTINATION transport type")
flags.BoolVar(&opts.scoped, "scoped", false, "Images at DESTINATION are prefix using the full source image path as scope")
flags.BoolVarP(&opts.all, "all", "a", false, "Copy all images if SOURCE-IMAGE is a list")
flags.AddFlagSet(&sharedFlags)
flags.AddFlagSet(&deprecatedTLSVerifyFlags)
flags.AddFlagSet(&srcFlags)
flags.AddFlagSet(&destFlags)
flags.AddFlagSet(&retryFlags)
return cmd
}
// UnmarshalYAML is the implementation of the Unmarshaler interface method
// unmarshalYAML is the implementation of the Unmarshaler interface method
// method for the tlsVerifyConfig type.
// It unmarshals the 'tls-verify' YAML key so that, when they key is not
// specified, tls verification is enforced.
@@ -211,6 +197,7 @@ func getImageTags(ctx context.Context, sysCtx *types.SystemContext, repoRef refe
// Some registries may decide to block the "list all tags" endpoint.
// Gracefully allow the sync to continue in this case.
logrus.Warnf("Registry disallows tag list retrieval: %s", err)
break
default:
return tags, errors.Wrapf(err, "Error determining repository tags for image %s", name)
}
@@ -243,7 +230,7 @@ func imagesToCopyFromRepo(sys *types.SystemContext, repoRef reference.Named) ([]
return sourceReferences, nil
}
// imagesToCopyFromDir builds a list of image references from the images found
// imagesTopCopyFromDir builds a list of image references from the images found
// in the source directory.
// It returns an image reference slice with as many elements as the images found
// and any error encountered.
@@ -273,11 +260,11 @@ func imagesToCopyFromDir(dirPath string) ([]types.ImageReference, error) {
return sourceReferences, nil
}
// imagesToCopyFromRegistry builds a list of repository descriptors from the images
// imagesTopCopyFromDir builds a list of repository descriptors from the images
// in a registry configuration.
// It returns a repository descriptors slice with as many elements as the images
// found and any error encountered. Each element of the slice is a list of
// image references, to be used as sync source.
// tagged image references, to be used as sync source.
func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourceCtx types.SystemContext) ([]repoDescriptor, error) {
serverCtx := &sourceCtx
// override ctx with per-registryName options
@@ -285,11 +272,10 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
serverCtx.DockerDaemonCertPath = cfg.CertDir
serverCtx.DockerDaemonInsecureSkipTLSVerify = (cfg.TLSVerify.skip == types.OptionalBoolTrue)
serverCtx.DockerInsecureSkipTLSVerify = cfg.TLSVerify.skip
if cfg.Credentials != (types.DockerAuthConfig{}) {
serverCtx.DockerAuthConfig = &cfg.Credentials
}
serverCtx.DockerAuthConfig = &cfg.Credentials
var repoDescList []repoDescriptor
for imageName, refs := range cfg.Images {
for imageName, tags := range cfg.Images {
repoLogger := logrus.WithFields(logrus.Fields{
"repo": imageName,
"registry": registryName,
@@ -304,37 +290,24 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
repoLogger.Info("Processing repo")
var sourceReferences []types.ImageReference
if len(refs) != 0 {
for _, ref := range refs {
tagLogger := logrus.WithFields(logrus.Fields{"ref": ref})
var named reference.Named
// first try as digest
if d, err := digest.Parse(ref); err == nil {
named, err = reference.WithDigest(repoRef, d)
if err != nil {
tagLogger.Error("Error processing ref, skipping")
logrus.Error(err)
continue
}
} else {
tagLogger.Debugf("Ref was not a digest, trying as a tag: %s", err)
named, err = reference.WithTag(repoRef, ref)
if err != nil {
tagLogger.Error("Error parsing ref, skipping")
logrus.Error(err)
continue
}
}
imageRef, err := docker.NewReference(named)
if len(tags) != 0 {
for _, tag := range tags {
tagLogger := logrus.WithFields(logrus.Fields{"tag": tag})
taggedRef, err := reference.WithTag(repoRef, tag)
if err != nil {
tagLogger.Error("Error processing ref, skipping")
tagLogger.Error("Error parsing tag, skipping")
logrus.Error(err)
continue
}
imageRef, err := docker.NewReference(taggedRef)
if err != nil {
tagLogger.Error("Error processing tag, skipping")
logrus.Errorf("Error getting image reference: %s", err)
continue
}
sourceReferences = append(sourceReferences, imageRef)
}
} else { // len(refs) == 0
} else { // len(tags) == 0
repoLogger.Info("Querying registry for image tags")
sourceReferences, err = imagesToCopyFromRepo(serverCtx, repoRef)
if err != nil {
@@ -345,12 +318,12 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
}
if len(sourceReferences) == 0 {
repoLogger.Warnf("No refs to sync found")
repoLogger.Warnf("No tags to sync found")
continue
}
repoDescList = append(repoDescList, repoDescriptor{
ImageRefs: sourceReferences,
Context: serverCtx})
TaggedImages: sourceReferences,
Context: serverCtx})
}
for imageName, tagRegex := range cfg.ImagesByTagRegex {
@@ -399,12 +372,12 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
}
if len(sourceReferences) == 0 {
repoLogger.Warnf("No refs to sync found")
repoLogger.Warnf("No tags to sync found")
continue
}
repoDescList = append(repoDescList, repoDescriptor{
ImageRefs: sourceReferences,
Context: serverCtx})
TaggedImages: sourceReferences,
Context: serverCtx})
}
return repoDescList, nil
@@ -437,13 +410,13 @@ func imagesToCopy(source string, transport string, sourceCtx *types.SystemContex
if err != nil {
return nil, errors.Wrapf(err, "Cannot obtain a valid image reference for transport %q and reference %q", docker.Transport.Name(), named.String())
}
desc.ImageRefs = []types.ImageReference{srcRef}
desc.TaggedImages = []types.ImageReference{srcRef}
} else {
desc.ImageRefs, err = imagesToCopyFromRepo(sourceCtx, named)
desc.TaggedImages, err = imagesToCopyFromRepo(sourceCtx, named)
if err != nil {
return descriptors, err
}
if len(desc.ImageRefs) == 0 {
if len(desc.TaggedImages) == 0 {
return descriptors, errors.Errorf("No images to sync found in %q", source)
}
}
@@ -459,11 +432,11 @@ func imagesToCopy(source string, transport string, sourceCtx *types.SystemContex
}
desc.DirBasePath = source
var err error
desc.ImageRefs, err = imagesToCopyFromDir(source)
desc.TaggedImages, err = imagesToCopyFromDir(source)
if err != nil {
return descriptors, err
}
if len(desc.ImageRefs) == 0 {
if len(desc.TaggedImages) == 0 {
return descriptors, errors.Errorf("No images to sync found in %q", source)
}
descriptors = append(descriptors, desc)
@@ -496,7 +469,6 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {
if len(args) != 2 {
return errorShouldDisplayUsage{errors.New("Exactly two arguments expected")}
}
opts.deprecatedTLSVerify.warnIfUsed([]string{"--src-tls-verify", "--dest-tls-verify"})
policyContext, err := opts.global.getPolicyContext()
if err != nil {
@@ -532,33 +504,14 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {
return errors.New("sync from 'dir' to 'dir' not implemented, consider using rsync instead")
}
imageListSelection := copy.CopySystemImage
if opts.all {
imageListSelection = copy.CopyAllImages
}
sourceCtx, err := opts.srcImage.newSystemContext()
if err != nil {
return err
}
var manifestType string
if opts.format.present {
manifestType, err = parseManifestFormat(opts.format.value)
if err != nil {
return err
}
}
ctx, cancel := opts.global.commandTimeoutContext()
defer cancel()
sourceArg := args[0]
var srcRepoList []repoDescriptor
if err = retry.RetryIfNecessary(ctx, func() error {
srcRepoList, err = imagesToCopy(sourceArg, opts.source, sourceCtx)
return err
}, opts.retryOpts); err != nil {
srcRepoList, err := imagesToCopy(sourceArg, opts.source, sourceCtx)
if err != nil {
return err
}
@@ -568,20 +521,20 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {
return err
}
ctx, cancel := opts.global.commandTimeoutContext()
defer cancel()
imagesNumber := 0
options := copy.Options{
RemoveSignatures: opts.removeSignatures,
SignBy: opts.signByFingerprint,
ReportWriter: os.Stdout,
DestinationCtx: destinationCtx,
ImageListSelection: imageListSelection,
OptimizeDestinationImageAlreadyExists: true,
ForceManifestMIMEType: manifestType,
RemoveSignatures: opts.removeSignatures,
SignBy: opts.signByFingerprint,
ReportWriter: os.Stdout,
DestinationCtx: destinationCtx,
}
for _, srcRepo := range srcRepoList {
options.SourceCtx = srcRepo.Context
for counter, ref := range srcRepo.ImageRefs {
for counter, ref := range srcRepo.TaggedImages {
var destSuffix string
switch ref.Transport() {
case docker.Transport:
@@ -608,13 +561,11 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {
logrus.WithFields(logrus.Fields{
"from": transports.ImageName(ref),
"to": transports.ImageName(destRef),
}).Infof("Copying image ref %d/%d", counter+1, len(srcRepo.ImageRefs))
}).Infof("Copying image tag %d/%d", counter+1, len(srcRepo.TaggedImages))
if err = retry.RetryIfNecessary(ctx, func() error {
_, err = copy.Image(ctx, policyContext, destRef, ref, &options)
return err
}, opts.retryOpts); err != nil {
return errors.Wrapf(err, "Error copying ref %q", transports.ImageName(ref))
_, err = copy.Image(ctx, policyContext, destRef, ref, &options)
if err != nil {
return errors.Wrapf(err, "Error copying tag %q", transports.ImageName(ref))
}
imagesNumber++
}

View File

@@ -1,8 +1,11 @@
//go:build !linux
// +build !linux
package main
func maybeReexec() error {
return nil
}
func reexecIfNecessaryForImages(inputImageNames ...string) error {
return nil
}

View File

@@ -2,19 +2,14 @@ package main
import (
"context"
"fmt"
"io"
"os"
"strings"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/compression"
"github.com/containers/image/v5/transports/alltransports"
"github.com/containers/image/v5/types"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
@@ -39,35 +34,6 @@ func commandAction(handler func(args []string, stdout io.Writer) error) func(cmd
}
}
// deprecatedTLSVerifyOption represents a deprecated --tls-verify option,
// which was accepted for all subcommands, for a time.
// Every user should call deprecatedTLSVerifyOption.warnIfUsed() as part of handling the CLI,
// whether or not the value actually ends up being used.
// DO NOT ADD ANY NEW USES OF THIS; just call dockerImageFlags with an appropriate, possibly empty, flagPrefix.
type deprecatedTLSVerifyOption struct {
tlsVerify optionalBool // FIXME FIXME: Warn if this is used, or even if it is ignored.
}
// warnIfUsed warns if tlsVerify was set by the user, and suggests alternatives (which should
// start with "--").
// Every user should call this as part of handling the CLI, whether or not the value actually
// ends up being used.
func (opts *deprecatedTLSVerifyOption) warnIfUsed(alternatives []string) {
if opts.tlsVerify.present {
logrus.Warnf("'--tls-verify' is deprecated, instead use: %s", strings.Join(alternatives, ", "))
}
}
// deprecatedTLSVerifyFlags prepares the CLI flag writing into deprecatedTLSVerifyOption, and the managed deprecatedTLSVerifyOption structure.
// DO NOT ADD ANY NEW USES OF THIS; just call dockerImageFlags with an appropriate, possibly empty, flagPrefix.
func deprecatedTLSVerifyFlags() (pflag.FlagSet, *deprecatedTLSVerifyOption) {
opts := deprecatedTLSVerifyOption{}
fs := pflag.FlagSet{}
flag := optionalBoolFlag(&fs, &opts.tlsVerify, "tls-verify", "require HTTPS and verify certificates when accessing the container registry (defaults to true)")
flag.Hidden = true
return fs, &opts
}
// sharedImageOptions collects CLI flags which are image-related, but do not change across images.
// This really should be a part of globalOptions, but that would break existing users of (skopeo copy --authfile=).
type sharedImageOptions struct {
@@ -86,15 +52,13 @@ func sharedImageFlags() (pflag.FlagSet, *sharedImageOptions) {
// the same across subcommands, but may be different for each image
// (e.g. may differ between the source and destination of a copy)
type dockerImageOptions struct {
global *globalOptions // May be shared across several imageOptions instances.
shared *sharedImageOptions // May be shared across several imageOptions instances.
deprecatedTLSVerify *deprecatedTLSVerifyOption // May be shared across several imageOptions instances, or nil.
authFilePath optionalString // Path to a */containers/auth.json (prefixed version to override shared image option).
credsOption optionalString // username[:password] for accessing a registry
registryToken optionalString // token to be used directly as a Bearer token when accessing the registry
dockerCertPath string // A directory using Docker-like *.{crt,cert,key} files for connecting to a registry or a daemon
tlsVerify optionalBool // Require HTTPS and verify certificates (for docker: and docker-daemon:)
noCreds bool // Access the registry anonymously
global *globalOptions // May be shared across several imageOptions instances.
shared *sharedImageOptions // May be shared across several imageOptions instances.
authFilePath optionalString // Path to a */containers/auth.json (prefixed version to override shared image option).
credsOption optionalString // username[:password] for accessing a registry
dockerCertPath string // A directory using Docker-like *.{crt,cert,key} files for connecting to a registry or a daemon
tlsVerify optionalBool // Require HTTPS and verify certificates (for docker: and docker-daemon:)
noCreds bool // Access the registry anonymously
}
// imageOptions collects CLI flags which are the same across subcommands, but may be different for each image
@@ -107,12 +71,11 @@ type imageOptions struct {
// dockerImageFlags prepares a collection of docker-transport specific CLI flags
// writing into imageOptions, and the managed imageOptions structure.
func dockerImageFlags(global *globalOptions, shared *sharedImageOptions, deprecatedTLSVerify *deprecatedTLSVerifyOption, flagPrefix, credsOptionAlias string) (pflag.FlagSet, *imageOptions) {
func dockerImageFlags(global *globalOptions, shared *sharedImageOptions, flagPrefix, credsOptionAlias string) (pflag.FlagSet, *imageOptions) {
flags := imageOptions{
dockerImageOptions: dockerImageOptions{
global: global,
shared: shared,
deprecatedTLSVerify: deprecatedTLSVerify,
global: global,
shared: shared,
},
}
@@ -128,7 +91,6 @@ func dockerImageFlags(global *globalOptions, shared *sharedImageOptions, depreca
f := fs.VarPF(newOptionalStringValue(&flags.credsOption), credsOptionAlias, "", "Use `USERNAME[:PASSWORD]` for accessing the registry")
f.Hidden = true
}
fs.Var(newOptionalStringValue(&flags.registryToken), flagPrefix+"registry-token", "Provide a Bearer token for accessing the registry")
fs.StringVar(&flags.dockerCertPath, flagPrefix+"cert-dir", "", "use certificates at `PATH` (*.crt, *.cert, *.key) to connect to the registry or daemon")
optionalBoolFlag(&fs, &flags.tlsVerify, flagPrefix+"tls-verify", "require HTTPS and verify certificates when talking to the container registry or daemon (defaults to true)")
fs.BoolVar(&flags.noCreds, flagPrefix+"no-creds", false, "Access the registry anonymously")
@@ -136,8 +98,8 @@ func dockerImageFlags(global *globalOptions, shared *sharedImageOptions, depreca
}
// imageFlags prepares a collection of CLI flags writing into imageOptions, and the managed imageOptions structure.
func imageFlags(global *globalOptions, shared *sharedImageOptions, deprecatedTLSVerify *deprecatedTLSVerifyOption, flagPrefix, credsOptionAlias string) (pflag.FlagSet, *imageOptions) {
dockerFlags, opts := dockerImageFlags(global, shared, deprecatedTLSVerify, flagPrefix, credsOptionAlias)
func imageFlags(global *globalOptions, shared *sharedImageOptions, flagPrefix, credsOptionAlias string) (pflag.FlagSet, *imageOptions) {
dockerFlags, opts := dockerImageFlags(global, shared, flagPrefix, credsOptionAlias)
fs := pflag.FlagSet{}
fs.StringVar(&opts.sharedBlobDir, flagPrefix+"shared-blob-dir", "", "`DIRECTORY` to use to share blobs across OCI repositories")
@@ -146,13 +108,6 @@ func imageFlags(global *globalOptions, shared *sharedImageOptions, deprecatedTLS
return fs, opts
}
func retryFlags() (pflag.FlagSet, *retry.RetryOptions) {
opts := retry.RetryOptions{}
fs := pflag.FlagSet{}
fs.IntVar(&opts.MaxRetry, "retry-times", 0, "the number of times to possibly retry")
return fs, &opts
}
// newSystemContext returns a *types.SystemContext corresponding to opts.
// It is guaranteed to return a fresh instance, so it is safe to make additional updates to it.
func (opts *imageOptions) newSystemContext() (*types.SystemContext, error) {
@@ -167,10 +122,6 @@ func (opts *imageOptions) newSystemContext() (*types.SystemContext, error) {
if opts.dockerImageOptions.authFilePath.present {
ctx.AuthFilePath = opts.dockerImageOptions.authFilePath.value
}
if opts.deprecatedTLSVerify != nil && opts.deprecatedTLSVerify.tlsVerify.present {
// If both this deprecated option and a non-deprecated option is present, we use the latter value.
ctx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!opts.deprecatedTLSVerify.tlsVerify.value)
}
if opts.tlsVerify.present {
ctx.DockerDaemonInsecureSkipTLSVerify = !opts.tlsVerify.value
}
@@ -187,9 +138,6 @@ func (opts *imageOptions) newSystemContext() (*types.SystemContext, error) {
return nil, err
}
}
if opts.registryToken.present {
ctx.DockerBearerRegistryToken = opts.registryToken.value
}
if opts.noCreds {
ctx.DockerAuthConfig = &types.DockerAuthConfig{}
}
@@ -197,24 +145,22 @@ func (opts *imageOptions) newSystemContext() (*types.SystemContext, error) {
return ctx, nil
}
// imageDestOptions is a superset of imageOptions specialized for image destinations.
// imageDestOptions is a superset of imageOptions specialized for iamge destinations.
type imageDestOptions struct {
*imageOptions
dirForceCompression bool // Compress layers when saving to the dir: transport
dirForceDecompression bool // Decompress layers when saving to the dir: transport
ociAcceptUncompressedLayers bool // Whether to accept uncompressed layers in the oci: transport
compressionFormat string // Format to use for the compression
compressionLevel optionalInt // Level to use for the compression
}
// imageDestFlags prepares a collection of CLI flags writing into imageDestOptions, and the managed imageDestOptions structure.
func imageDestFlags(global *globalOptions, shared *sharedImageOptions, deprecatedTLSVerify *deprecatedTLSVerifyOption, flagPrefix, credsOptionAlias string) (pflag.FlagSet, *imageDestOptions) {
genericFlags, genericOptions := imageFlags(global, shared, deprecatedTLSVerify, flagPrefix, credsOptionAlias)
func imageDestFlags(global *globalOptions, shared *sharedImageOptions, flagPrefix, credsOptionAlias string) (pflag.FlagSet, *imageDestOptions) {
genericFlags, genericOptions := imageFlags(global, shared, flagPrefix, credsOptionAlias)
opts := imageDestOptions{imageOptions: genericOptions}
fs := pflag.FlagSet{}
fs.AddFlagSet(&genericFlags)
fs.BoolVar(&opts.dirForceCompression, flagPrefix+"compress", false, "Compress tarball image layers when saving to directory using the 'dir' transport. (default is same compression type as source)")
fs.BoolVar(&opts.dirForceDecompression, flagPrefix+"decompress", false, "Decompress tarball image layers when saving to directory using the 'dir' transport. (default is same compression type as source)")
fs.BoolVar(&opts.ociAcceptUncompressedLayers, flagPrefix+"oci-accept-uncompressed-layers", false, "Allow uncompressed image layers when saving to an OCI image using the 'oci' transport. (default is to compress things that aren't compressed)")
fs.StringVar(&opts.compressionFormat, flagPrefix+"compress-format", "", "`FORMAT` to use for the compression")
fs.Var(newOptionalIntValue(&opts.compressionLevel), flagPrefix+"compress-level", "`LEVEL` to use for the compression")
@@ -230,7 +176,6 @@ func (opts *imageDestOptions) newSystemContext() (*types.SystemContext, error) {
}
ctx.DirForceCompress = opts.dirForceCompression
ctx.DirForceDecompress = opts.dirForceDecompression
ctx.OCIAcceptUncompressedLayers = opts.ociAcceptUncompressedLayers
if opts.compressionFormat != "" {
cf, err := compression.AlgorithmByName(opts.compressionFormat)
@@ -284,21 +229,6 @@ func parseImageSource(ctx context.Context, opts *imageOptions, name string) (typ
return ref.NewImageSource(ctx, sys)
}
// parseManifestFormat parses format parameter for copy and sync command.
// It returns string value to use as manifest MIME type
func parseManifestFormat(manifestFormat string) (string, error) {
switch manifestFormat {
case "oci":
return imgspecv1.MediaTypeImageManifest, nil
case "v2s1":
return manifest.DockerV2Schema1SignedMediaType, nil
case "v2s2":
return manifest.DockerV2Schema2MediaType, nil
default:
return "", fmt.Errorf("unknown format %q. Choose one of the supported formats: 'oci', 'v2s1', or 'v2s2'", manifestFormat)
}
}
// usageTemplate returns the usage template for skopeo commands
// This blocks the displaying of the global options. The main skopeo
// command should not use this.

View File

@@ -4,11 +4,8 @@ import (
"os"
"testing"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/types"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -18,26 +15,17 @@ func fakeGlobalOptions(t *testing.T, flags []string) (*globalOptions, *cobra.Com
app, opts := createApp()
cmd := &cobra.Command{}
app.AddCommand(cmd)
err := app.ParseFlags(flags)
err := cmd.ParseFlags(flags)
require.NoError(t, err)
return opts, cmd
}
// fakeImageOptions creates imageOptions and sets it according to globalFlags/cmdFlags.
func fakeImageOptions(t *testing.T, flagPrefix string, useDeprecatedTLSVerify bool,
globalFlags []string, cmdFlags []string) *imageOptions {
func fakeImageOptions(t *testing.T, flagPrefix string, globalFlags []string, cmdFlags []string) *imageOptions {
globalOpts, cmd := fakeGlobalOptions(t, globalFlags)
sharedFlags, sharedOpts := sharedImageFlags()
var deprecatedTLSVerifyFlag pflag.FlagSet
var deprecatedTLSVerifyOpt *deprecatedTLSVerifyOption
if useDeprecatedTLSVerify {
deprecatedTLSVerifyFlag, deprecatedTLSVerifyOpt = deprecatedTLSVerifyFlags()
}
imageFlags, imageOpts := imageFlags(globalOpts, sharedOpts, deprecatedTLSVerifyOpt, flagPrefix, "")
imageFlags, imageOpts := imageFlags(globalOpts, sharedOpts, flagPrefix, "")
cmd.Flags().AddFlagSet(&sharedFlags)
if useDeprecatedTLSVerify {
cmd.Flags().AddFlagSet(&deprecatedTLSVerifyFlag)
}
cmd.Flags().AddFlagSet(&imageFlags)
err := cmd.ParseFlags(cmdFlags)
require.NoError(t, err)
@@ -46,15 +34,13 @@ func fakeImageOptions(t *testing.T, flagPrefix string, useDeprecatedTLSVerify bo
func TestImageOptionsNewSystemContext(t *testing.T) {
// Default state
opts := fakeImageOptions(t, "dest-", true, []string{}, []string{})
opts := fakeImageOptions(t, "dest-", []string{}, []string{})
res, err := opts.newSystemContext()
require.NoError(t, err)
assert.Equal(t, &types.SystemContext{
DockerRegistryUserAgent: defaultUserAgent,
}, res)
assert.Equal(t, &types.SystemContext{}, res)
// Set everything to non-default values.
opts = fakeImageOptions(t, "dest-", true, []string{
opts = fakeImageOptions(t, "dest-", []string{
"--registries.d", "/srv/registries.d",
"--override-arch", "overridden-arch",
"--override-os", "overridden-os",
@@ -68,7 +54,6 @@ func TestImageOptionsNewSystemContext(t *testing.T) {
"--dest-daemon-host", "daemon-host.example.com",
"--dest-tls-verify=false",
"--dest-creds", "creds-user:creds-password",
"--dest-registry-token", "faketoken",
})
res, err = opts.newSystemContext()
require.NoError(t, err)
@@ -82,37 +67,55 @@ func TestImageOptionsNewSystemContext(t *testing.T) {
DockerCertPath: "/srv/cert-dir",
DockerInsecureSkipTLSVerify: types.OptionalBoolTrue,
DockerAuthConfig: &types.DockerAuthConfig{Username: "creds-user", Password: "creds-password"},
DockerBearerRegistryToken: "faketoken",
DockerDaemonCertPath: "/srv/cert-dir",
DockerDaemonHost: "daemon-host.example.com",
DockerDaemonInsecureSkipTLSVerify: true,
DockerRegistryUserAgent: defaultUserAgent,
BigFilesTemporaryDir: "/srv",
}, res)
// Global/per-command tlsVerify behavior is tested in TestTLSVerifyFlags.
// Global/per-command tlsVerify behavior
for _, c := range []struct {
global, cmd string
expectedDocker types.OptionalBool
expectedDockerDaemon bool
}{
{"", "", types.OptionalBoolUndefined, false},
{"", "false", types.OptionalBoolTrue, true},
{"", "true", types.OptionalBoolFalse, false},
{"false", "", types.OptionalBoolTrue, false},
{"false", "false", types.OptionalBoolTrue, true},
{"false", "true", types.OptionalBoolFalse, false},
{"true", "", types.OptionalBoolFalse, false},
{"true", "false", types.OptionalBoolTrue, true},
{"true", "true", types.OptionalBoolFalse, false},
} {
globalFlags := []string{}
if c.global != "" {
globalFlags = append(globalFlags, "--tls-verify="+c.global)
}
cmdFlags := []string{}
if c.cmd != "" {
cmdFlags = append(cmdFlags, "--dest-tls-verify="+c.cmd)
}
opts := fakeImageOptions(t, "dest-", globalFlags, cmdFlags)
res, err = opts.newSystemContext()
require.NoError(t, err)
assert.Equal(t, c.expectedDocker, res.DockerInsecureSkipTLSVerify, "%#v", c)
assert.Equal(t, c.expectedDockerDaemon, res.DockerDaemonInsecureSkipTLSVerify, "%#v", c)
}
// Invalid option values
opts = fakeImageOptions(t, "dest-", true, []string{}, []string{"--dest-creds", ""})
opts = fakeImageOptions(t, "dest-", []string{}, []string{"--dest-creds", ""})
_, err = opts.newSystemContext()
assert.Error(t, err)
}
// fakeImageDestOptions creates imageDestOptions and sets it according to globalFlags/cmdFlags.
func fakeImageDestOptions(t *testing.T, flagPrefix string, useDeprecatedTLSVerify bool,
globalFlags []string, cmdFlags []string) *imageDestOptions {
func fakeImageDestOptions(t *testing.T, flagPrefix string, globalFlags []string, cmdFlags []string) *imageDestOptions {
globalOpts, cmd := fakeGlobalOptions(t, globalFlags)
sharedFlags, sharedOpts := sharedImageFlags()
var deprecatedTLSVerifyFlag pflag.FlagSet
var deprecatedTLSVerifyOpt *deprecatedTLSVerifyOption
if useDeprecatedTLSVerify {
deprecatedTLSVerifyFlag, deprecatedTLSVerifyOpt = deprecatedTLSVerifyFlags()
}
imageFlags, imageOpts := imageDestFlags(globalOpts, sharedOpts, deprecatedTLSVerifyOpt, flagPrefix, "")
imageFlags, imageOpts := imageDestFlags(globalOpts, sharedOpts, flagPrefix, "")
cmd.Flags().AddFlagSet(&sharedFlags)
if useDeprecatedTLSVerify {
cmd.Flags().AddFlagSet(&deprecatedTLSVerifyFlag)
}
cmd.Flags().AddFlagSet(&imageFlags)
err := cmd.ParseFlags(cmdFlags)
require.NoError(t, err)
@@ -121,12 +124,10 @@ func fakeImageDestOptions(t *testing.T, flagPrefix string, useDeprecatedTLSVerif
func TestImageDestOptionsNewSystemContext(t *testing.T) {
// Default state
opts := fakeImageDestOptions(t, "dest-", true, []string{}, []string{})
opts := fakeImageDestOptions(t, "dest-", []string{}, []string{})
res, err := opts.newSystemContext()
require.NoError(t, err)
assert.Equal(t, &types.SystemContext{
DockerRegistryUserAgent: defaultUserAgent,
}, res)
assert.Equal(t, &types.SystemContext{}, res)
oldXRD, hasXRD := os.LookupEnv("REGISTRY_AUTH_FILE")
defer func() {
@@ -141,18 +142,15 @@ func TestImageDestOptionsNewSystemContext(t *testing.T) {
os.Setenv("REGISTRY_AUTH_FILE", authFile)
// Explicitly set everything to default, except for when the default is “not present”
opts = fakeImageDestOptions(t, "dest-", true, []string{}, []string{
opts = fakeImageDestOptions(t, "dest-", []string{}, []string{
"--dest-compress=false",
})
res, err = opts.newSystemContext()
require.NoError(t, err)
assert.Equal(t, &types.SystemContext{
AuthFilePath: authFile,
DockerRegistryUserAgent: defaultUserAgent,
}, res)
assert.Equal(t, &types.SystemContext{AuthFilePath: authFile}, res)
// Set everything to non-default values.
opts = fakeImageDestOptions(t, "dest-", true, []string{
opts = fakeImageDestOptions(t, "dest-", []string{
"--registries.d", "/srv/registries.d",
"--override-arch", "overridden-arch",
"--override-os", "overridden-os",
@@ -166,7 +164,6 @@ func TestImageDestOptionsNewSystemContext(t *testing.T) {
"--dest-daemon-host", "daemon-host.example.com",
"--dest-tls-verify=false",
"--dest-creds", "creds-user:creds-password",
"--dest-registry-token", "faketoken",
})
res, err = opts.newSystemContext()
require.NoError(t, err)
@@ -180,145 +177,19 @@ func TestImageDestOptionsNewSystemContext(t *testing.T) {
DockerCertPath: "/srv/cert-dir",
DockerInsecureSkipTLSVerify: types.OptionalBoolTrue,
DockerAuthConfig: &types.DockerAuthConfig{Username: "creds-user", Password: "creds-password"},
DockerBearerRegistryToken: "faketoken",
DockerDaemonCertPath: "/srv/cert-dir",
DockerDaemonHost: "daemon-host.example.com",
DockerDaemonInsecureSkipTLSVerify: true,
DockerRegistryUserAgent: defaultUserAgent,
DirForceCompress: true,
BigFilesTemporaryDir: "/srv",
}, res)
// Global/per-command tlsVerify behavior is tested in TestTLSVerifyFlags.
// Invalid option values in imageOptions
opts = fakeImageDestOptions(t, "dest-", true, []string{}, []string{"--dest-creds", ""})
opts = fakeImageDestOptions(t, "dest-", []string{}, []string{"--dest-creds", ""})
_, err = opts.newSystemContext()
assert.Error(t, err)
}
func TestTLSVerifyFlags(t *testing.T) {
type systemContextOpts interface { // Either *imageOptions or *imageDestOptions
newSystemContext() (*types.SystemContext, error)
}
for _, creator := range []struct {
name string
newOpts func(useDeprecatedTLSVerify bool, globalFlags, cmdFlags []string) systemContextOpts
}{
{
"imageFlags",
func(useDeprecatedTLSVerify bool, globalFlags, cmdFlags []string) systemContextOpts {
return fakeImageOptions(t, "dest-", useDeprecatedTLSVerify, globalFlags, cmdFlags)
},
},
{
"imageDestFlags",
func(useDeprecatedTLSVerify bool, globalFlags, cmdFlags []string) systemContextOpts {
return fakeImageDestOptions(t, "dest-", useDeprecatedTLSVerify, globalFlags, cmdFlags)
},
},
} {
t.Run(creator.name, func(t *testing.T) {
for _, c := range []struct {
global, deprecatedCmd, cmd string
expectedDocker types.OptionalBool
expectedDockerDaemon bool
}{
{"", "", "", types.OptionalBoolUndefined, false},
{"", "", "false", types.OptionalBoolTrue, true},
{"", "", "true", types.OptionalBoolFalse, false},
{"", "false", "", types.OptionalBoolTrue, false},
{"", "false", "false", types.OptionalBoolTrue, true},
{"", "false", "true", types.OptionalBoolFalse, false},
{"", "true", "", types.OptionalBoolFalse, false},
{"", "true", "false", types.OptionalBoolTrue, true},
{"", "true", "true", types.OptionalBoolFalse, false},
{"false", "", "", types.OptionalBoolTrue, false},
{"false", "", "false", types.OptionalBoolTrue, true},
{"false", "", "true", types.OptionalBoolFalse, false},
{"false", "false", "", types.OptionalBoolTrue, false},
{"false", "false", "false", types.OptionalBoolTrue, true},
{"false", "false", "true", types.OptionalBoolFalse, false},
{"false", "true", "", types.OptionalBoolFalse, false},
{"false", "true", "false", types.OptionalBoolTrue, true},
{"false", "true", "true", types.OptionalBoolFalse, false},
{"true", "", "", types.OptionalBoolFalse, false},
{"true", "", "false", types.OptionalBoolTrue, true},
{"true", "", "true", types.OptionalBoolFalse, false},
{"true", "false", "", types.OptionalBoolTrue, false},
{"true", "false", "false", types.OptionalBoolTrue, true},
{"true", "false", "true", types.OptionalBoolFalse, false},
{"true", "true", "", types.OptionalBoolFalse, false},
{"true", "true", "false", types.OptionalBoolTrue, true},
{"true", "true", "true", types.OptionalBoolFalse, false},
} {
globalFlags := []string{}
if c.global != "" {
globalFlags = append(globalFlags, "--tls-verify="+c.global)
}
cmdFlags := []string{}
if c.deprecatedCmd != "" {
cmdFlags = append(cmdFlags, "--tls-verify="+c.deprecatedCmd)
}
if c.cmd != "" {
cmdFlags = append(cmdFlags, "--dest-tls-verify="+c.cmd)
}
opts := creator.newOpts(true, globalFlags, cmdFlags)
res, err := opts.newSystemContext()
require.NoError(t, err)
assert.Equal(t, c.expectedDocker, res.DockerInsecureSkipTLSVerify, "%#v", c)
assert.Equal(t, c.expectedDockerDaemon, res.DockerDaemonInsecureSkipTLSVerify, "%#v", c)
if c.deprecatedCmd == "" { // Test also the behavior when deprecatedTLSFlag is not recognized
// Use globalFlags from the previous test
cmdFlags := []string{}
if c.cmd != "" {
cmdFlags = append(cmdFlags, "--dest-tls-verify="+c.cmd)
}
opts := creator.newOpts(false, globalFlags, cmdFlags)
res, err = opts.newSystemContext()
require.NoError(t, err)
assert.Equal(t, c.expectedDocker, res.DockerInsecureSkipTLSVerify, "%#v", c)
assert.Equal(t, c.expectedDockerDaemon, res.DockerDaemonInsecureSkipTLSVerify, "%#v", c)
}
}
})
}
}
func TestParseManifestFormat(t *testing.T) {
for _, testCase := range []struct {
formatParam string
expectedManifestType string
expectErr bool
}{
{"oci",
imgspecv1.MediaTypeImageManifest,
false},
{"v2s1",
manifest.DockerV2Schema1SignedMediaType,
false},
{"v2s2",
manifest.DockerV2Schema2MediaType,
false},
{"",
"",
true},
{"badValue",
"",
true},
} {
manifestType, err := parseManifestFormat(testCase.formatParam)
if testCase.expectErr {
require.Error(t, err)
} else {
require.NoError(t, err)
}
assert.Equal(t, manifestType, testCase.expectedManifestType)
}
}
// since there is a shared authfile image option and a non-shared (prefixed) one, make sure the override logic
// works correctly.
func TestImageOptionsAuthfileOverride(t *testing.T) {
@@ -353,13 +224,12 @@ func TestImageOptionsAuthfileOverride(t *testing.T) {
}, "/srv/dest-authfile",
},
} {
opts := fakeImageOptions(t, testCase.flagPrefix, false, []string{}, testCase.cmdFlags)
opts := fakeImageOptions(t, testCase.flagPrefix, []string{}, testCase.cmdFlags)
res, err := opts.newSystemContext()
require.NoError(t, err)
assert.Equal(t, &types.SystemContext{
AuthFilePath: testCase.expectedAuthfilePath,
DockerRegistryUserAgent: defaultUserAgent,
AuthFilePath: testCase.expectedAuthfilePath,
}, res)
}
}

View File

@@ -49,14 +49,11 @@ _skopeo_copy() {
--dest-tls-verify
--src-daemon-host
--dest-daemon-host
--src-registry-token
--dest-registry-token
"
local boolean_options="
--all
--dest-compress
--dest-decompress
--remove-signatures
--src-no-creds
--dest-no-creds
@@ -71,50 +68,11 @@ _skopeo_copy() {
_complete_ "$options_with_args" "$boolean_options" "$transports"
}
_skopeo_sync() {
local options_with_args="
--authfile
--dest
--dest-authfile
--dest-cert-
--dest-creds
--dest-registry-token string
--format
--retry-times
--sign-by
--src
--src-authfile
--src-cert-dir
--src-creds
--src-registry-token
"
local boolean_options="
--all
--dest-no-creds
--dest-tls-verify
--remove-signatures
--scoped
--src-no-creds
--src-tls-verify
"
local transports
transports="
$(_skopeo_supported_transports "${FUNCNAME//"_skopeo_"/}")
"
_complete_ "$options_with_args" "$boolean_options" "$transports"
}
_skopeo_inspect() {
local options_with_args="
--authfile
--creds
--cert-dir
--format
--retry-times
--registry-token
"
local boolean_options="
--config
@@ -161,7 +119,6 @@ _skopeo_delete() {
--authfile
--creds
--cert-dir
--registry-token
"
local boolean_options="
--tls-verify
@@ -178,14 +135,11 @@ _skopeo_delete() {
_skopeo_layers() {
local options_with_args="
--authfile
--creds
--cert-dir
--registry-token
"
local boolean_options="
--tls-verify
--no-creds
"
_complete_ "$options_with_args" "$boolean_options"
}
@@ -195,7 +149,6 @@ _skopeo_list_repository_tags() {
--authfile
--creds
--cert-dir
--registry-token
"
local boolean_options="
@@ -233,7 +186,7 @@ _skopeo_logout() {
}
_skopeo_skopeo() {
# XXX: Changes here need to be reflected in the manually expanded
# XXX: Changes here need to be refleceted in the manually expanded
# string in the `case` statement below as well.
local options_with_args="
--policy
@@ -267,7 +220,7 @@ _skopeo_skopeo() {
)
case "$prev" in
# XXX: Changes here need to be reflected in $options_with_args as well.
# XXX: Changes here need to be refleceted in $options_with_args as well.
--policy|--registries.d|--override-arch|--override-os|--override-variant|--command-timeout)
return
;;
@@ -297,7 +250,7 @@ _cli_bash_autocomplete() {
local counter=1
while [ $counter -lt "$cword" ]; do
case "${words[$counter]}" in
skopeo|copy|sync|inspect|delete|manifest-digest|standalone-sign|standalone-verify|help|h|list-repository-tags)
skopeo|copy|inspect|delete|manifest-digest|standalone-sign|standalone-verify|help|h|list-repository-tags)
command="${words[$counter]//-/_}"
cpos=$counter
(( cpos++ ))

View File

@@ -1,95 +0,0 @@
#!/bin/bash
# This script is intended to be executed by automation or humans
# under a hack/get_ci_vm.sh context. Use under any other circumstances
# is unlikely to function.
set -e
if [[ -r "/etc/automation_environment" ]]; then
source /etc/automation_environment
source $AUTOMATION_LIB_PATH/common_lib.sh
else
(
echo "WARNING: It does not appear that containers/automation was installed."
echo " Functionality of most of ${BASH_SOURCE[0]} will be negatively"
echo " impacted."
) > /dev/stderr
fi
OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
# GCE image-name compatible string representation of distribution _major_ version
OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | tr -d '.')"
# Combined to ease some usage
OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
export "PATH=$PATH:$GOPATH/bin"
podmanmake() {
req_env_vars GOPATH SKOPEO_PATH SKOPEO_CI_CONTAINER_FQIN
warn "Accumulated technical-debt requires execution inside a --privileged container. This is very likely hiding bugs!"
showrun podman run -it --rm --privileged \
-e GOPATH=$GOPATH \
-v $GOPATH:$GOPATH:Z \
-w $SKOPEO_PATH \
$SKOPEO_CI_CONTAINER_FQIN \
make "$@"
}
_run_setup() {
if [[ "$OS_RELEASE_ID" == "fedora" ]]; then
# This is required as part of the standard Fedora VM setup
growpart /dev/sda 1
resize2fs /dev/sda1
# VM's come with the distro. skopeo pre-installed
dnf erase -y skopeo
else
die "Unknown/unsupported distro. $OS_REL_VER"
fi
}
_run_vendor() {
podmanmake vendor BUILDTAGS="$BUILDTAGS"
}
_run_build() {
make bin/skopeo BUILDTAGS="$BUILDTAGS"
}
_run_cross() {
podmanmake local-cross BUILDTAGS="$BUILDTAGS"
}
_run_doccheck() {
make validate-docs BUILDTAGS="$BUILDTAGS"
}
_run_unit() {
podmanmake test-unit-local BUILDTAGS="$BUILDTAGS"
}
_run_integration() {
podmanmake test-integration-local BUILDTAGS="$BUILDTAGS"
}
_run_system() {
# Ensure we start with a clean-slate
podman system reset --force
# Executes with containers required for testing.
showrun make test-system-local BUILDTAGS="$BUILDTAGS"
}
req_env_vars SKOPEO_PATH BUILDTAGS
handler="_run_${1}"
if [ "$(type -t $handler)" != "function" ]; then
die "Unknown/Unsupported command-line argument '$1'"
fi
msg "************************************************************"
msg "Runner executing $1 on $OS_REL_VER"
msg "************************************************************"
cd "$SKOPEO_PATH"
$handler

View File

@@ -6,39 +6,19 @@
## Overview
This directory contains the Dockerfiles necessary to create the skopeoimage container
images that are housed on quay.io under the skopeo account. All repositories where
the images live are public and can be pulled without credentials. These container images are secured and the
resulting containers can run safely with privileges within the container.
The container images are built using the latest Fedora and then Skopeo is installed into them.
The PATH in the container images is set to the default PATH provided by Fedora. Also, the
ENTRYPOINT and the WORKDIR variables are not set within these container images, as such they
default to `/`.
The container images are:
* `quay.io/containers/skopeo:<version>` and `quay.io/skopeo/stable:<version>` -
These images are built when a new Skopeo version becomes available in
Fedora. These images are intended to be unchanging and stable, they will
never be updated by automation once they've been pushed. For build details,
please [see the configuration file](stable/Dockerfile).
* `quay.io/containers/skopeo:latest` and `quay.io/skopeo/stable:latest` -
Built daily using the same Dockerfile as above. The skopeo version
will remain the "latest" available in Fedora, however the image
contents may vary compared to the version-tagged images.
* `quay.io/skopeo/testing:latest` - This image is built daily, using the
latest version of Skopeo that was in the Fedora `updates-testing` repository.
The image is Built with [the testing Dockerfile](testing/Dockerfile).
* `quay.io/skopeo/upstream:latest` - This image is built daily using the latest
code found in this GitHub repository. Due to the image changing frequently,
it's not guaranteed to be stable or even executable. The image is built with
[the upstream Dockerfile](upstream/Dockerfile).
This directory contains the Dockerfiles necessary to create the three skopeoimage container
images that are housed on quay.io under the skopeo account. All three repositories where
the images live are public and can be pulled without credentials. These container images
are secured and the resulting containers can run safely. The container images are built
using the latest Fedora and then Skopeo is installed into them:
* quay.io/skopeo/stable - This image is built using the latest stable version of Skopeo in a Fedora based container. Built with skopeoimage/stable/Dockerfile.
* quay.io/skopeo/upstream - This image is built using the latest code found in this GitHub repository. When someone creates a commit and pushes it, the image is created. Due to that the image changes frequently and is not guaranteed to be stable. Built with skopeoimage/upstream/Dockerfile.
* quay.io/skopeo/testing - This image is built using the latest version of Skopeo that is or was in updates testing for Fedora. At times this may be the same as the stable image. This container image will primarily be used by the development teams for verification testing when a new package is created. Built with skopeoimage/testing/Dockerfile.
## Sample Usage
Although not required, it is suggested that [Podman](https://github.com/containers/podman) be used with these container images.
Although not required, it is suggested that [Podman](https://github.com/containers/libpod) be used with these container images.
```
# Get Help on Skopeo

View File

@@ -6,14 +6,14 @@
# This image can be used to create a secured container
# that runs safely with privileges within the container.
#
FROM registry.fedoraproject.org/fedora:33
FROM registry.fedoraproject.org/fedora:32
# Don't include container-selinux and remove
# directories used by yum that are just taking
# up space. Also reinstall shadow-utils as without
# doing so, the setuid/setgid bits on newuidmap
# and newgidmap are lost in the Fedora images.
RUN useradd skopeo; yum -y update; yum -y reinstall shadow-utils; yum -y install skopeo fuse-overlayfs --exclude container-selinux; yum -y clean all; rm -rf /var/cache/dnf/* /var/log/dnf* /var/log/yum*
RUN useradd skopeo; yum -y update; yum -y reinstall shadow-utils; yum -y install skopeo fuse-overlayfs --exclude container-selinux; yum clean all; rm -rf /var/cache /var/log/dnf* /var/log/yum.*;
# Adjust storage.conf to enable Fuse storage.
RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' /etc/containers/storage.conf
@@ -27,7 +27,7 @@ RUN echo skopeo:100000:65536 > /etc/subuid
RUN echo skopeo:100000:65536 > /etc/subgid
# Point to the Authorization file
ENV REGISTRY_AUTH_FILE=/tmp/auth.json
ENV REGISTRY_AUTH_FILE=/auth.json
# Set the entrypoint
ENTRYPOINT ["/usr/bin/skopeo"]

View File

@@ -7,14 +7,14 @@
# This image can be used to create a secured container
# that runs safely with privileges within the container.
#
FROM registry.fedoraproject.org/fedora:33
FROM registry.fedoraproject.org/fedora:32
# Don't include container-selinux and remove
# directories used by yum that are just taking
# up space. Also reinstall shadow-utils as without
# doing so, the setuid/setgid bits on newuidmap
# and newgidmap are lost in the Fedora images.
RUN useradd skopeo; yum -y update; yum -y reinstall shadow-utils; yum -y install skopeo fuse-overlayfs --enablerepo updates-testing --exclude container-selinux; yum -y clean all; rm -rf /var/cache/dnf/* /var/log/dnf* /var/log/yum*
RUN useradd skopeo; yum -y update; yum -y reinstall shadow-utils; yum -y install skopeo fuse-overlayfs --enablerepo updates-testing --exclude container-selinux; yum clean all; rm -rf /var/cache /var/log/dnf* /var/log/yum.*;
# Adjust storage.conf to enable Fuse storage.
RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' /etc/containers/storage.conf
@@ -28,7 +28,7 @@ RUN echo skopeo:100000:65536 > /etc/subuid
RUN echo skopeo:100000:65536 > /etc/subgid
# Point to the Authorization file
ENV REGISTRY_AUTH_FILE=/tmp/auth.json
ENV REGISTRY_AUTH_FILE=/auth.json
# Set the entrypoint
ENTRYPOINT ["/usr/bin/skopeo"]

View File

@@ -6,7 +6,7 @@
# This image can be used to create a secured container
# that runs safely with privileges within the container.
#
FROM registry.fedoraproject.org/fedora:33
FROM registry.fedoraproject.org/fedora:32
# Don't include container-selinux and remove
# directories used by yum that are just taking
@@ -29,12 +29,11 @@ mkdir /root/skopeo; \
git clone https://github.com/containers/skopeo /root/skopeo/src/github.com/containers/skopeo; \
export GOPATH=/root/skopeo; \
cd /root/skopeo/src/github.com/containers/skopeo; \
make bin/skopeo;\
make PREFIX=/usr install;\
make binary-local;\
make install;\
rm -rf /root/skopeo/*; \
yum -y remove git golang go-md2man make; \
yum -y clean all; yum -y clean all; rm -rf /var/cache/dnf/* /var/log/dnf* /var/log/yum*
yum clean all; rm -rf /var/cache /var/log/dnf* /var/log/yum.*;
# Adjust storage.conf to enable Fuse storage.
RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' /etc/containers/storage.conf
@@ -48,7 +47,7 @@ RUN echo skopeo:100000:65536 > /etc/subuid
RUN echo skopeo:100000:65536 > /etc/subgid
# Point to the Authorization file
ENV REGISTRY_AUTH_FILE=/tmp/auth.json
ENV REGISTRY_AUTH_FILE=/auth.json
# Set the entrypoint
ENTRYPOINT ["/usr/bin/skopeo"]

View File

@@ -4,7 +4,7 @@
skopeo\-copy - Copy an image (manifest, filesystem layers, signatures) from one location to another.
## SYNOPSIS
**skopeo copy** [*options*] _source-image_ _destination-image_
**skopeo copy** [**--sign-by=**_key-ID_] _source-image destination-image_
## DESCRIPTION
Copy an image (manifest, filesystem layers, signatures) from one location to another.
@@ -15,16 +15,9 @@ Uses the system's trust policy to validate images, rejects images not trusted by
_destination-image_ use the "image name" format described above
_source-image_ and _destination-image_ are interpreted completely independently; e.g. the destination name does not
automatically inherit any parts of the source name.
## OPTIONS
**--additional-tag**=_strings_
Additional tags (supports docker-archive).
**--all**, **-a**
**--all**
If _source-image_ refers to a list of images, instead of copying just the image which matches the current OS and
architecture (subject to the use of the global --override-os, --override-arch and --override-variant options), attempt to copy all of
@@ -46,131 +39,50 @@ Path of the authentication file for the source registry. Uses path given by `--a
Path of the authentication file for the destination registry. Uses path given by `--authfile`, if not provided.
**--dest-shared-blob-dir** _directory_
**--format, -f** _manifest-type_ Manifest type (oci, v2s1, or v2s2) to use when saving image to directory using the 'dir:' transport (default is manifest type of source)
Directory to use to share blobs across OCI repositories.
**--quiet, -q** suppress output information when copying images
**--digestfile** _path_
**--remove-signatures** do not copy signatures, if any, from _source-image_. Necessary when copying a signed image to a destination which does not support signatures.
After copying the image, write the digest of the resulting image to the file.
**--sign-by=**_key-id_ add a signature using that key ID for an image name corresponding to _destination-image_
**--encrypt-layer** _ints_
**--encryption-key** _Key_ a reference prefixed with the encryption protocol to use. The supported protocols are JWE, PGP and PKCS7. For instance, jwe:/path/to/key.pem or pgp:admin@example.com or pkcs7:/path/to/x509-file. This feature is still *experimental*.
*Experimental* the 0-indexed layer indices, with support for negative indexing (e.g. 0 is the first layer, -1 is the last layer)
**--decryption-key** _Key_ a reference required to perform decryption of container images. This should point to files which represent keys and/or certificates that can be used for decryption. Decryption will be tried with all keys. This feature is still *experimental*.
**--format**, **-f** _manifest-type_
**--src-creds** _username[:password]_ for accessing the source registry
MANIFEST TYPE (oci, v2s1, or v2s2) to use in the destination (default is manifest type of source, with fallbacks)
**--dest-compress** _bool-value_ Compress tarball image layers when saving to directory using the 'dir' transport. (default is same compression type as source)
**--help**, **-h**
**--dest-oci-accept-uncompressed-layers** _bool-value_ Allow uncompressed image layers when saving to an OCI image using the 'oci' transport. (default is to compress things that aren't compressed)
Print usage statement
**--dest-creds** _username[:password]_ for accessing the destination registry
**--quiet**, **-q**
**--src-cert-dir** _path_ Use certificates at _path_ (*.crt, *.cert, *.key) to connect to the source registry or daemon
Suppress output information when copying images.
**--src-no-creds** _bool-value_ Access the registry anonymously.
**--remove-signatures**
**--src-tls-verify** _bool-value_ Require HTTPS and verify certificates when talking to container source registry or daemon (defaults to true)
Do not copy signatures, if any, from _source-image_. Necessary when copying a signed image to a destination which does not support signatures.
**--dest-cert-dir** _path_ Use certificates at _path_ (*.crt, *.cert, *.key) to connect to the destination registry or daemon
**--sign-by**=_key-id_
**--dest-no-creds** _bool-value_ Access the registry anonymously.
Add a signature using that key ID for an image name corresponding to _destination-image_
**--dest-tls-verify** _bool-value_ Require HTTPS and verify certificates when talking to container destination registry or daemon (defaults to true)
**--src-shared-blob-dir** _directory_
**--src-daemon-host** _host_ Copy from docker daemon at _host_. If _host_ starts with `tcp://`, HTTPS is enabled by default. To use plain HTTP, use the form `http://` (default is `unix:///var/run/docker.sock`).
Directory to use to share blobs across OCI repositories.
**--encryption-key** _protocol:keyfile_
Specifies the encryption protocol, which can be JWE (RFC7516), PGP (RFC4880), and PKCS7 (RFC2315) and the key material required for image encryption. For instance, jwe:/path/to/key.pem or pgp:admin@example.com or pkcs7:/path/to/x509-file.
**--decryption-key** _key[:passphrase]_
Key to be used for decryption of images. Key can point to keys and/or certificates. Decryption will be tried with all keys. If the key is protected by a passphrase, it is required to be passed in the argument and omitted otherwise.
**--src-creds** _username[:password]_
Credentials for accessing the source registry.
**--dest-compress** _bool-value_
Compress tarball image layers when saving to directory using the 'dir' transport. (default is same compression type as source).
**--dest-decompress** _bool-value_
Decompress tarball image layers when saving to directory using the 'dir' transport. (default is same compression type as source).
**--dest-oci-accept-uncompressed-layers** _bool-value_
Allow uncompressed image layers when saving to an OCI image using the 'oci' transport. (default is to compress things that aren't compressed).
**--dest-creds** _username[:password]_
Credentials for accessing the destination registry.
**--src-cert-dir** _path_
Use certificates at _path_ (*.crt, *.cert, *.key) to connect to the source registry or daemon.
**--src-no-creds** _bool-value_
Access the registry anonymously.
**--src-tls-verify** _bool-value_
Require HTTPS and verify certificates when talking to container source registry or daemon (defaults to true).
**--dest-cert-dir** _path_
Use certificates at _path_ (*.crt, *.cert, *.key) to connect to the destination registry or daemon.
**--dest-no-creds** _bool-value_
Access the registry anonymously.
**--dest-tls-verify** _bool-value_
Require HTTPS and verify certificates when talking to container destination registry or daemon (defaults to true).
**--src-daemon-host** _host_
Copy from docker daemon at _host_. If _host_ starts with `tcp://`, HTTPS is enabled by default. To use plain HTTP, use the form `http://` (default is `unix:///var/run/docker.sock`).
**--dest-daemon-host** _host_
Copy to docker daemon at _host_. If _host_ starts with `tcp://`, HTTPS is enabled by default. To use plain HTTP, use the form `http://` (default is `unix:///var/run/docker.sock`).
**--dest-daemon-host** _host_ Copy to docker daemon at _host_. If _host_ starts with `tcp://`, HTTPS is enabled by default. To use plain HTTP, use the form `http://` (default is `unix:///var/run/docker.sock`).
Existing signatures, if any, are preserved as well.
**--dest-compress-format** _format_
**--dest-compress-format** _format_ Specifies the compression format to use. Supported values are: `gzip` and `zstd`.
Specifies the compression format to use. Supported values are: `gzip` and `zstd`.
**--dest-compress-level** _format_
Specifies the compression level to use. The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive), while for gzip it is 1-9 (inclusive).
**--src-registry-token** _token_
Bearer token for accessing the source registry.
**--dest-registry-token** _token_
Bearer token for accessing the destination registry.
**--retry-times**
The number of times to retry. Retry wait time will be exponentially increased based on the number of failed attempts.
**--dest-compress-level** _format_ Specifies the compression level to use. The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive), while for gzip it is 1-9 (inclusive).
## EXAMPLES
To just copy an image from one registry to another:
```sh
$ skopeo copy docker://quay.io/skopeo/stable:latest docker://registry.example.com/skopeo:latest
```
To copy the layers of the docker.io busybox image to a local directory:
```sh
$ mkdir -p /var/lib/images/busybox
@@ -220,8 +132,9 @@ skopeo copy --encryption-key jwe:./public.key --encrypt-layer 1 oci:local_nginx
```
## SEE ALSO
skopeo(1), skopeo-login(1), docker-login(1), containers-auth.json(5), containers-policy.json(5), containers-transports(5), containers-signature(5)
skopeo(1), skopeo-login(1), docker-login(1), containers-auth.json(5), containers-policy.json(5), containers-transports(5)
## AUTHORS
Antonio Murdaca <runcom@redhat.com>, Miloslav Trmac <mitr@redhat.com>, Jhon Honce <jhonce@redhat.com>

View File

@@ -1,10 +1,10 @@
% skopeo-delete(1)
## NAME
skopeo\-delete - Mark the _image-name_ for later deletion by the registry's garbage collector.
skopeo\-delete - Mark _image-name_ for deletion.
## SYNOPSIS
**skopeo delete** [*options*] _image-name_
**skopeo delete** _image-name_
Mark _image-name_ for deletion. To release the allocated disk space, you must login to the container registry server and execute the container registry garbage collector. E.g.,
@@ -19,51 +19,21 @@ $ docker exec -it registry /usr/bin/registry garbage-collect /etc/docker-distrib
```
## OPTIONS
**--authfile** _path_
Path of the authentication file. Default is ${XDG_RUNTIME\_DIR}/containers/auth.json, which is set using `skopeo login`.
If the authorization state is not found there, $HOME/.docker/config.json is checked, which is set using `docker login`.
Path of the authentication file. Default is ${XDG_RUNTIME\_DIR}/containers/auth.json, which is set using `skopeo login`.
If the authorization state is not found there, $HOME/.docker/config.json is checked, which is set using `docker login`.
**--creds** _username[:password]_
**--creds** _username[:password]_ for accessing the registry
Credentials for accessing the registry.
**--cert-dir** _path_ Use certificates at _path_ (*.crt, *.cert, *.key) to connect to the registry
**--cert-dir** _path_
**--tls-verify** _bool-value_ Require HTTPS and verify certificates when talking to container registries (defaults to true)
Use certificates at _path_ (*.crt, *.cert, *.key) to connect to the registry.
**--daemon-host** _host_
Use docker daemon host at _host_ (`docker-daemon:` transport only)
**--help**, **-h**
Print usage statement
**--no-creds** _bool-value_
Access the registry anonymously.
**--no-creds** _bool-value_ Access the registry anonymously.
Additionally, the registry must allow deletions by setting `REGISTRY_STORAGE_DELETE_ENABLED=true` for the registry daemon.
**--registry-token** _token_
Bearer token for accessing the registry.
**--retry-times**
The number of times to retry. Retry wait time will be exponentially increased based on the number of failed attempts.
**--shared-blob-dir** _directory_
Directory to use to share blobs across OCI repositories.
**--tls-verify**=_bool_
Require HTTPS and verify certificates when talking to the container registry or daemon (defaults to true)
## EXAMPLES
Mark image example/pause for deletion from the registry.example.com registry:
@@ -79,3 +49,4 @@ skopeo(1), skopeo-login(1), docker-login(1), containers-auth.json(5)
## AUTHORS
Antonio Murdaca <runcom@redhat.com>, Miloslav Trmac <mitr@redhat.com>, Jhon Honce <jhonce@redhat.com>

View File

@@ -1,73 +1,37 @@
% skopeo-inspect(1)
## NAME
skopeo\-inspect - Return low-level information about _image-name_ in a registry.
skopeo\-inspect - Return low-level information about _image-name_ in a registry
## SYNOPSIS
**skopeo inspect** [*options*] _image-name_
## DESCRIPTION
**skopeo inspect** [**--raw**] [**--config**] _image-name_
Return low-level information about _image-name_ in a registry
_image-name_ name of image to retrieve information about
**--raw** output raw manifest, default is to format in JSON
## OPTIONS
_image-name_ name of image to retrieve information about
**--authfile** _path_
**--config** output configuration in OCI format, default is to format in JSON
Path of the authentication file. Default is ${XDG\_RUNTIME\_DIR}/containers/auth.json, which is set using `skopeo login`.
If the authorization state is not found there, $HOME/.docker/config.json is checked, which is set using `docker login`.
_image-name_ name of image to retrieve configuration for
**--cert-dir** _path_
**--config** **--raw** output configuration in raw format
Use certificates at _path_ (\*.crt, \*.cert, \*.key) to connect to the registry.
_image-name_ name of image to retrieve configuration for
**--config**
**--authfile** _path_
Output configuration in OCI format, default is to format in JSON format.
Path of the authentication file. Default is ${XDG\_RUNTIME\_DIR}/containers/auth.json, which is set using `skopeo login`.
If the authorization state is not found there, $HOME/.docker/config.json is checked, which is set using `docker login`.
**--creds** _username[:password]_
**--creds** _username[:password]_ for accessing the registry
Username and password for accessing the registry.
**--cert-dir** _path_ Use certificates at _path_ (\*.crt, \*.cert, \*.key) to connect to the registry
**--daemon-host** _host_
**--tls-verify** _bool-value_ Require HTTPS and verify certificates when talking to container registries (defaults to true)
Use docker daemon host at _host_ (`docker-daemon:` transport only)
**--format**, **-f**=*format*
Format the output using the given Go template.
The keys of the returned JSON can be used as the values for the --format flag (see examples below).
**--help**, **-h**
Print usage statement
**--no-creds**
Access the registry anonymously.
**--raw**
Output raw manifest or config data depending on --config option.
The --format option is not supported with --raw option.
**--registry-token** _Bearer token_
Registry token for accessing the registry.
**--retry-times**
The number of times to retry; retry wait time will be exponentially increased based on the number of failed attempts.
**--shared-blob-dir** _directory_
Directory to use to share blobs across OCI repositories.
**--tls-verify**=_bool_
Require HTTPS and verify certificates when talking to the container registry or daemon (defaults to true)
**--no-creds** _bool-value_ Access the registry anonymously.
## EXAMPLES
@@ -78,14 +42,14 @@ $ skopeo inspect docker://docker.io/fedora
"Name": "docker.io/library/fedora",
"Digest": "sha256:a97914edb6ba15deb5c5acf87bd6bd5b6b0408c96f48a5cbd450b5b04509bb7d",
"RepoTags": [
"20",
"21",
"22",
"23",
"24",
"heisenbug",
"latest",
"rawhide"
"20",
"21",
"22",
"23",
"24",
"heisenbug",
"latest",
"rawhide"
],
"Created": "2016-06-20T19:33:43.220526898Z",
"DockerVersion": "1.10.3",
@@ -93,24 +57,15 @@ $ skopeo inspect docker://docker.io/fedora
"Architecture": "amd64",
"Os": "linux",
"Layers": [
"sha256:7c91a140e7a1025c3bc3aace4c80c0d9933ac4ee24b8630a6b0b5d8b9ce6b9d4"
"sha256:7c91a140e7a1025c3bc3aace4c80c0d9933ac4ee24b8630a6b0b5d8b9ce6b9d4"
]
}
```
```
$ /bin/skopeo inspect --config docker://registry.fedoraproject.org/fedora --format "{{ .Architecture }}"
amd64
```
```
$ /bin/skopeo inspect --format '{{ .Env }}' docker://registry.access.redhat.com/ubi8
[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin container=oci]
```
# SEE ALSO
skopeo(1), skopeo-login(1), docker-login(1), containers-auth.json(5)
## AUTHORS
Antonio Murdaca <runcom@redhat.com>, Miloslav Trmac <mitr@redhat.com>, Jhon Honce <jhonce@redhat.com>

View File

@@ -1,47 +1,27 @@
% skopeo-list-tags(1)
## NAME
skopeo\-list\-tags - List tags in the transport-specific image repository.
skopeo\-list\-tags - Return a list of tags the transport-specific image repository
## SYNOPSIS
**skopeo list-tags** [*options*] _repository-name_
**skopeo list-tags** _repository-name_
Return a list of tags from _repository-name_ in a registry.
_repository-name_ name of repository to retrieve tag listing from
## OPTIONS
**--authfile** _path_
**--authfile** _path_
Path of the authentication file. Default is ${XDG\_RUNTIME\_DIR}/containers/auth.json, which is set using `skopeo login`.
Path of the authentication file. Default is ${XDG\_RUNTIME\_DIR}/containers/auth.json, which is set using `skopeo login`.
If the authorization state is not found there, $HOME/.docker/config.json is checked, which is set using `docker login`.
**--creds** _username[:password]_ for accessing the registry.
**--creds** _username[:password]_ for accessing the registry
**--cert-dir** _path_
**--cert-dir** _path_ Use certificates at _path_ (\*.crt, \*.cert, \*.key) to connect to the registry
Use certificates at _path_ (\*.crt, \*.cert, \*.key) to connect to the registry.
**--tls-verify** _bool-value_ Require HTTPS and verify certificates when talking to container registries (defaults to true)
**--help**, **-h**
Print usage statement
**--no-creds** _bool-value_
Access the registry anonymously.
**--registry-token** _Bearer token_
Bearer token for accessing the registry.
**--retry-times**
The number of times to retry. Retry wait time will be exponentially increased based on the number of failed attempts.
**--tls-verify**=_bool_
Require HTTPS and verify certificates when talking to the container registry or daemon (defaults to true)
**--no-creds** _bool-value_ Access the registry anonymously.
## REPOSITORY NAMES
@@ -52,18 +32,18 @@ This commands refers to repositories using a _transport_`:`_details_ format. The
**docker://**_docker-repository-reference_
A repository in a registry implementing the "Docker Registry HTTP API V2". By default, uses the authorization state in either `$XDG_RUNTIME_DIR/containers/auth.json`, which is set using `(skopeo login)`. If the authorization state is not found there, `$HOME/.docker/config.json` is checked, which is set using `(docker login)`.
A _docker-repository-reference_ is of the form: **registryhost:port/repositoryname** which is similar to an _image-reference_ but with no tag or digest allowed as the last component (e.g no `:latest` or `@sha256:xyz`)
Examples of valid docker-repository-references:
"docker.io/myuser/myrepo"
"docker.io/nginx"
"docker.io/library/fedora"
"localhost:5000/myrepository"
Examples of invalid references:
"docker.io/nginx:latest"
"docker.io/myuser/myimage:v1.0"
"docker.io/myuser/myimage@sha256:f48c4cc192f4c3c6a069cb5cca6d0a9e34d6076ba7c214fd0cc3ca60e0af76bb"
## EXAMPLES
@@ -119,3 +99,4 @@ skopeo(1), skopeo-login(1), docker-login(1), containers-auth.json(5)
## AUTHORS
Zach Hill <zach@anchore.com>

View File

@@ -1,10 +1,10 @@
% skopeo-login(1)
## NAME
skopeo\-login - Login to a container registry.
skopeo\-login - Login to a container registry
## SYNOPSIS
**skopeo login** [*options*] _registry_
**skoepo login** [*options*] *registry*
## DESCRIPTION
**skopeo login** logs into a specified registry server with the correct username
@@ -43,18 +43,16 @@ Return the logged-in user for the registry. Return error if no login is found.
Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry.
Default certificates directory is _/etc/containers/certs.d_.
**--tls-verify**=*true|false*
Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true,
then TLS verification will be used. If set to false, then TLS verification will not be used. If not specified,
TLS verification will be used unless the target registry is listed as an insecure registry in registries.conf.
**--help**, **-h**
Print usage statement
**--tls-verify**=_bool_
Require HTTPS and verify certificates when talking to the container registry or daemon (defaults to true)
**--verbose**, **-v**
Write more detailed information to stdout
## EXAMPLES
```

View File

@@ -1,10 +1,10 @@
% skopeo-logout(1)
## NAME
skopeo\-logout - Logout of a container registry.
skopeo\-logout - Logout of a container registry
## SYNOPSIS
**skopeo logout** [*options*] _registry_
**skopeo logout** [*options*] *registry*
## DESCRIPTION
**skopeo logout** logs out of a specified registry server by deleting the cached credentials
@@ -29,10 +29,6 @@ Remove the cached credentials for all registries in the auth file
Print usage statement
**--tls-verify**=_bool_
Require HTTPS and verify certificates when talking to the container registry or daemon (defaults to true)
## EXAMPLES
```

View File

@@ -1,7 +1,7 @@
% skopeo-manifest-digest(1)
## NAME
skopeo\-manifest\-digest - Compute a manifest digest for a manifest-file and write it to standard output.
skopeo\-manifest\-digest -Compute a manifest digest of manifest-file and write it to standard output.
## SYNOPSIS
**skopeo manifest-digest** _manifest-file_
@@ -10,12 +10,6 @@ skopeo\-manifest\-digest - Compute a manifest digest for a manifest-file and wri
Compute a manifest digest of _manifest-file_ and write it to standard output.
## OPTIONS
**--help**, **-h**
Print usage statement
## EXAMPLES
```sh
@@ -29,3 +23,4 @@ skopeo(1)
## AUTHORS
Antonio Murdaca <runcom@redhat.com>, Miloslav Trmac <mitr@redhat.com>, Jhon Honce <jhonce@redhat.com>

View File

@@ -1,13 +1,14 @@
% skopeo-standalone-sign(1)
## NAME
skopeo\-standalone-sign - Debugging tool - Publish and sign an image in one step.
skopeo\-standalone-sign - Simple Sign an image
## SYNOPSIS
**skopeo standalone-sign** [*options*] _manifest_ _docker-reference_ _key-fingerprint_ **--output**|**-o** _signature_
**skopeo standalone-sign** _manifest docker-reference key-fingerprint_ **--output**|**-o** _signature_
## DESCRIPTION
This is primarily a debugging tool, useful for special cases, and usually should not be a part of your normal operational workflow; use `skopeo copy --sign-by` instead to publish and sign an image in one step.
This is primarily a debugging tool, or useful for special cases,
and usually should not be a part of your normal operational workflow; use `skopeo copy --sign-by` instead to publish and sign an image in one step.
_manifest_ Path to a file containing the image manifest
@@ -15,15 +16,7 @@ This is primarily a debugging tool, useful for special cases, and usually should
_key-fingerprint_ Key identity to use for signing
## OPTIONS
**--help**, **-h**
Print usage statement
**--output**, **-o** _output file_
Write signature to _output file_.
**--output**|**-o** output file
## EXAMPLES
@@ -32,13 +25,10 @@ $ skopeo standalone-sign busybox-manifest.json registry.example.com/example/busy
$
```
## NOTES
This command is intended for use with local signatures e.g. OpenPGP ( other signature formats may be added in the future ), as per containers-signature(5). Furthermore, this command does **not** interact with the artifacts generated by Docker Content Trust (DCT). For more information, please see [containers-signature(5)](https://github.com/containers/image/blob/main/docs/containers-signature.5.md).
## SEE ALSO
skopeo(1), skopeo-copy(1), containers-signature(5)
## AUTHORS
Antonio Murdaca <runcom@redhat.com>, Miloslav Trmac <mitr@redhat.com>, Jhon Honce <jhonce@redhat.com>

View File

@@ -1,16 +1,14 @@
% skopeo-standalone-verify(1)
## NAME
skopeo\-standalone\-verify - Verify an image signature.
skopeo\-standalone\-verify - Verify an image signature
## SYNOPSIS
**skopeo standalone-verify** _manifest_ _docker-reference_ _key-fingerprint_ _signature_
**skopeo standalone-verify** _manifest docker-reference key-fingerprint signature_
## DESCRIPTION
Verify a signature using local files; the digest will be printed on success. This is primarily a debugging tool, useful for special cases,
and usually should not be a part of your normal operational workflow. Additionally, consider configuring a signature verification policy file,
as per containers-policy.json(5).
Verify a signature using local files, digest will be printed on success.
_manifest_ Path to a file containing the image manifest
@@ -22,12 +20,6 @@ as per containers-policy.json(5).
**Note:** If you do use this, make sure that the image can not be changed at the source location between the times of its verification and use.
## OPTIONS
**--help**, **-h**
Print usage statement
## EXAMPLES
```sh
@@ -35,13 +27,10 @@ $ skopeo standalone-verify busybox-manifest.json registry.example.com/example/bu
Signature verified, digest sha256:20bf21ed457b390829cdbeec8795a7bea1626991fda603e0d01b4e7f60427e55
```
## NOTES
This command is intended for use with local signatures e.g. OpenPGP ( other signature formats may be added in the future ), as per containers-signature(5). Furthermore, this command does **not** interact with the artifacts generated by Docker Content Trust (DCT). For more information, please see [containers-signature(5)](https://github.com/containers/image/blob/main/docs/containers-signature.5.md).
## SEE ALSO
skopeo(1), containers-signature(5), containers-policy.json(5)
skopeo(1), containers-signature(5)
## AUTHORS
Antonio Murdaca <runcom@redhat.com>, Miloslav Trmac <mitr@redhat.com>, Jhon Honce <jhonce@redhat.com>

View File

@@ -5,7 +5,7 @@ skopeo\-sync - Synchronize images between container registries and local directo
## SYNOPSIS
**skopeo sync** [*options*] --src _transport_ --dest _transport_ _source_ _destination_
**skopeo sync** --src _transport_ --dest _transport_ _source_ _destination_
## DESCRIPTION
Synchronize images between container registries and local directories.
@@ -32,11 +32,6 @@ When the `--scoped` option is specified, images are prefixed with the source ima
name can be stored at _destination_.
## OPTIONS
**--all**, **-a**
If one of the images in __src__ refers to a list of images, instead of copying just the image which matches the current OS and
architecture (subject to the use of the global --override-os, --override-arch and --override-variant options), attempt to copy all of
the images in the list, and the list itself.
**--authfile** _path_
Path of the authentication file. Default is ${XDG\_RUNTIME\_DIR}/containers/auth.json, which is set using `skopeo login`.
@@ -50,21 +45,15 @@ Path of the authentication file for the source registry. Uses path given by `--a
Path of the authentication file for the destination registry. Uses path given by `--authfile`, if not provided.
**--src**, **-s** _transport_ Transport for the source repository.
**--src** _transport_ Transport for the source repository.
**--dest**, **-d** _transport_ Destination transport.
**--format**, **-f** _manifest-type_ Manifest Type (oci, v2s1, or v2s2) to use when syncing image(s) to a destination (default is manifest type of source, with fallbacks).
**--help**, **-h**
Print usage statement.
**--dest** _transport_ Destination transport.
**--scoped** Prefix images with the source image path, so that multiple images with the same name can be stored at _destination_.
**--remove-signatures** Do not copy signatures, if any, from _source-image_. This is necessary when copying a signed image to a destination which does not support signatures.
**--sign-by**=_key-id_ Add a signature using that key ID for an image name corresponding to _destination-image_.
**--sign-by=**_key-id_ Add a signature using that key ID for an image name corresponding to _destination-image_.
**--src-creds** _username[:password]_ for accessing the source registry.
@@ -82,12 +71,6 @@ Print usage statement.
**--dest-tls-verify** _bool-value_ Require HTTPS and verify certificates when talking to a container destination registry or daemon (defaults to true).
**--src-registry-token** _Bearer token_ for accessing the source registry.
**--dest-registry-token** _Bearer token_ for accessing the destination registry.
**--retry-times** the number of times to retry, retry wait time will be exponentially increased based on the number of failed attempts.
## EXAMPLES
### Synchronizing to a local directory
@@ -160,7 +143,6 @@ registry.example.com:
redis:
- "1.0"
- "2.0"
- "sha256:0000000000000000000000000000000011111111111111111111111111111111"
images-by-tag-regex:
nginx: ^1\.13\.[12]-alpine-perl$
credentials:
@@ -180,14 +162,14 @@ skopeo sync --src yaml --dest docker sync.yml my-registry.local.lan/repo/
```
This will copy the following images:
- Repository `registry.example.com/busybox`: all images, as no tags are specified.
- Repository `registry.example.com/redis`: images tagged "1.0" and "2.0" along with image with digest "sha256:0000000000000000000000000000000011111111111111111111111111111111".
- Repository `registry.example.com/redis`: images tagged "1.0" and "2.0".
- Repository `registry.example.com/nginx`: images tagged "1.13.1-alpine-perl" and "1.13.2-alpine-perl".
- Repository `quay.io/coreos/etcd`: images tagged "latest".
For the registry `registry.example.com`, the "john"/"this is a secret" credentials are used, with server TLS certificates located at `/home/john/certs`.
TLS verification is normally enabled, and it can be disabled setting `tls-verify` to `false`.
In the above example, TLS verification is enabled for `registry.example.com`, while is
In the above example, TLS verification is enabled for `reigstry.example.com`, while is
disabled for `quay.io`.
## SEE ALSO

View File

@@ -44,71 +44,44 @@ Most commands refer to container images, using a _transport_`:`_details_ format.
**oci:**_path_**:**_tag_
An image _tag_ in a directory compliant with "Open Container Image Layout Specification" at _path_.
**oci-archive:**_path_**:**_tag_
An image _tag_ in a tar archive compliant with "Open Container Image Layout Specification" at _path_.
See [containers-transports(5)](https://github.com/containers/image/blob/master/docs/containers-transports.5.md) for details.
## OPTIONS
**--command-timeout** _duration_
**--command-timeout** _duration_ Timeout for the command execution.
Timeout for the command execution.
**--debug** enable debug output
**--debug**
**--help**|**-h** Show help
enable debug output
**--insecure-policy** Adopt an insecure, permissive policy that allows anything. This obviates the need for a policy file.
**--help**, **-h**
**--override-arch** _arch_ Use _arch_ instead of the architecture of the machine for choosing images.
Show help
**--override-os** _OS_ Use _OS_ instead of the running OS for choosing images.
**--insecure-policy**
**--override-variant** _VARIANT_ Use _VARIANT_ instead of the running architecture variant for choosing images.
Adopt an insecure, permissive policy that allows anything. This obviates the need for a policy file.
**--policy** _path-to-policy_ Path to a policy.json file to use for verifying signatures and deciding whether an image is trusted, overriding the default trust policy file.
**--override-arch** _arch_
**--registries.d** _dir_ use registry configuration files in _dir_ (e.g. for container signature storage), overriding the default path.
Use _arch_ instead of the architecture of the machine for choosing images.
**--tmpdir** _dir_ used to store temporary files. Defaults to /var/tmp.
**--override-os** _os_
Use _OS_ instead of the running OS for choosing images.
**--override-variant** _variant_
Use _variant_ instead of the running architecture variant for choosing images.
**--policy** _path-to-policy_
Path to a policy.json file to use for verifying signatures and deciding whether an image is trusted, overriding the default trust policy file.
**--registries.d** _dir_
Use registry configuration files in _dir_ (e.g. for container signature storage), overriding the default path.
**--tmpdir** _dir_
Directory used to store temporary files. Defaults to /var/tmp.
**--version**, **-v**
Print the version number
**--version**|**-v** print the version number
## COMMANDS
| Command | Description |
| ----------------------------------------- | ------------------------------------------------------------------------------ |
| [skopeo-copy(1)](skopeo-copy.1.md) | Copy an image (manifest, filesystem layers, signatures) from one location to another. |
| [skopeo-delete(1)](skopeo-delete.1.md) | Mark the _image-name_ for later deletion by the registry's garbage collector. |
| [skopeo-inspect(1)](skopeo-inspect.1.md) | Return low-level information about _image-name_ in a registry. |
| [skopeo-list-tags(1)](skopeo-list-tags.1.md) | List tags in the transport-specific image repository. |
| [skopeo-delete(1)](skopeo-delete.1.md) | Mark image-name for deletion. |
| [skopeo-inspect(1)](skopeo-inspect.1.md) | Return low-level information about image-name in a registry. |
| [skopeo-list-tags(1)](skopeo-list-tags.1.md) | List the tags for the given transport/repository. |
| [skopeo-login(1)](skopeo-login.1.md) | Login to a container registry. |
| [skopeo-logout(1)](skopeo-logout.1.md) | Logout of a container registry. |
| [skopeo-manifest-digest(1)](skopeo-manifest-digest.1.md) | Compute a manifest digest for a manifest-file and write it to standard output. |
| [skopeo-standalone-sign(1)](skopeo-standalone-sign.1.md) | Debugging tool - Publish and sign an image in one step. |
| [skopeo-standalone-verify(1)](skopeo-standalone-verify.1.md)| Verify an image signature. |
| [skopeo-sync(1)](skopeo-sync.1.md)| Synchronize images between container registries and local directories. |
| [skopeo-manifest-digest(1)](skopeo-manifest-digest.1.md) | Compute a manifest digest of manifest-file and write it to standard output.|
| [skopeo-standalone-sign(1)](skopeo-standalone-sign.1.md) | Sign an image. |
| [skopeo-standalone-verify(1)](skopeo-standalone-verify.1.md)| Verify an image. |
| [skopeo-sync(1)](skopeo-sync.1.md)| Copy images from one or more repositories to a user specified destination. |
## FILES
**/etc/containers/policy.json**

25
go.mod
View File

@@ -3,24 +3,25 @@ module github.com/containers/skopeo
go 1.12
require (
github.com/containers/common v0.42.1
github.com/containers/image/v5 v5.15.2
github.com/containers/ocicrypt v1.1.2
github.com/containers/storage v1.34.1
github.com/docker/docker v20.10.8+incompatible
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
github.com/containers/common v0.14.0
github.com/containers/image/v5 v5.5.1
github.com/containers/ocicrypt v1.0.2
github.com/containers/storage v1.20.2
github.com/docker/docker v1.4.2-0.20191219165747-a9416c67da9f
github.com/dsnet/compress v0.0.1 // indirect
github.com/go-check/check v0.0.0-20180628173108-788fd7840127
github.com/onsi/gomega v1.15.0 // indirect
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6
github.com/opencontainers/image-tools v0.0.0-20170926011501-6d941547fa1d
github.com/opencontainers/runtime-spec v1.0.0 // indirect
github.com/pkg/errors v0.9.1
github.com/russross/blackfriday v2.0.0+incompatible // indirect
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.2.1
github.com/sirupsen/logrus v1.6.0
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.7.0
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
github.com/stretchr/testify v1.6.1
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2
go4.org v0.0.0-20190218023631-ce4c26f7be8e // indirect
gopkg.in/yaml.v2 v2.4.0
golang.org/x/text v0.3.3 // indirect
gopkg.in/yaml.v2 v2.3.0
)

1106
go.sum

File diff suppressed because it is too large Load Diff

View File

@@ -1,61 +0,0 @@
#!/usr/bin/env bash
#
# For help and usage information, simply execute the script w/o any arguments.
#
# This script is intended to be run by Red Hat skopeo developers who need
# to debug problems specifically related to Cirrus-CI automated testing.
# It requires that you have been granted prior access to create VMs in
# google-cloud. For non-Red Hat contributors, VMs are available as-needed,
# with supervision upon request.
set -e
SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}")
SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH")
REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../")
# Help detect if we were called by get_ci_vm container
GET_CI_VM="${GET_CI_VM:-0}"
in_get_ci_vm() {
if ((GET_CI_VM==0)); then
echo "Error: $1 is not intended for use in this context"
exit 2
fi
}
# get_ci_vm APIv1 container entrypoint calls into this script
# to obtain required repo. specific configuration options.
if [[ "$1" == "--config" ]]; then
in_get_ci_vm "$1"
cat <<EOF
DESTDIR="/var/tmp/go/src/github.com/containers/skopeo"
UPSTREAM_REPO="https://github.com/containers/skopeo.git"
GCLOUD_PROJECT="skopeo"
GCLOUD_IMGPROJECT="libpod-218412"
GCLOUD_CFG="skopeo"
GCLOUD_ZONE="${GCLOUD_ZONE:-us-central1-f}"
GCLOUD_CPUS="2"
GCLOUD_MEMORY="4Gb"
GCLOUD_DISK="200"
EOF
elif [[ "$1" == "--setup" ]]; then
in_get_ci_vm "$1"
# get_ci_vm container entrypoint calls us with this option on the
# Cirrus-CI environment instance, to perform repo.-specific setup.
echo "+ Executing setup" > /dev/stderr
${GOSRC}/${SCRIPT_BASE}/runner.sh setup
else
# Create and access VM for specified Cirrus-CI task
mkdir -p $HOME/.config/gcloud/ssh
podman run -it --rm \
--tz=local \
-e NAME="$USER" \
-e SRCDIR=/src \
-e GCLOUD_ZONE="$GCLOUD_ZONE" \
-e DEBUG="${DEBUG:-0}" \
-v $REPO_DIRPATH:/src:O \
-v $HOME/.config/gcloud:/root/.config/gcloud:z \
-v $HOME/.config/gcloud/ssh:/root/.ssh:z \
quay.io/libpod/get_ci_vm:latest "$@"
fi

View File

@@ -25,8 +25,12 @@ export MAKEDIR="$SCRIPTDIR/make"
# We're a nice, sexy, little shell script, and people might try to run us;
# but really, they shouldn't. We want to be in a container!
# The magic value is defined inside our Dockerfile.
if [[ "$container_magic" != "85531765-346b-4316-bdb8-358e4cca9e5d" ]]; then
inContainer="AssumeSoInitially"
if [ "$PWD" != "/go/src/$SKOPEO_PKG" ]; then
unset inContainer
fi
if [ -z "$inContainer" ]; then
{
echo "# WARNING! I don't seem to be running in a Docker container."
echo "# The result of this command might be an incorrect build, and will not be"
@@ -35,9 +39,6 @@ if [[ "$container_magic" != "85531765-346b-4316-bdb8-358e4cca9e5d" ]]; then
echo "# Try this instead: make all"
echo "#"
} >&2
else
echo "# I appear to be running inside my designated container image, good!"
export SKOPEO_CONTAINER_TESTS=1
fi
echo

View File

@@ -5,7 +5,7 @@ if [ -z "$VALIDATE_UPSTREAM" ]; then
# are running more than one validate bundlescript
VALIDATE_REPO='https://github.com/containers/skopeo.git'
VALIDATE_BRANCH='main'
VALIDATE_BRANCH='master'
if [ "$TRAVIS" = 'true' -a "$TRAVIS_PULL_REQUEST" != 'false' ]; then
VALIDATE_REPO="https://github.com/${TRAVIS_REPO_SLUG}.git"

View File

@@ -8,7 +8,7 @@ bundle_test_integration() {
# subshell so that we can export PATH without breaking other things
(
make bin/skopeo ${BUILDTAGS:+BUILDTAGS="$BUILDTAGS"}
make PREFIX=/usr install
make binary-local ${BUILDTAGS:+BUILDTAGS="$BUILDTAGS"}
make install
bundle_test_integration
) 2>&1

View File

@@ -11,8 +11,8 @@ sed -i \
/etc/containers/storage.conf
# Build skopeo, install into /usr/bin
make bin/skopeo ${BUILDTAGS:+BUILDTAGS="$BUILDTAGS"}
make PREFIX=/usr install
make binary-local ${BUILDTAGS:+BUILDTAGS="$BUILDTAGS"}
make install
# Run tests
SKOPEO_BINARY=/usr/bin/skopeo bats --tap systemtest

View File

@@ -1,150 +0,0 @@
#!/usr/bin/env bash
#
# man-page-checker - validate and cross-reference man page names
#
# This is the script that cross-checks BETWEEN MAN PAGES. It is not the
# script that cross-checks that each option in skopeo foo --help is listed
# in skopeo-foo.1.md and vice-versa; that one is xref-helpmsgs-manpages.
#
verbose=
for i; do
case "$i" in
-v|--verbose) verbose=verbose ;;
esac
done
die() {
echo "$(basename $0): $*" >&2
exit 1
}
cd $(dirname $0)/../docs || die "Please run me from top-level skopeo dir"
rc=0
# Pass 1: cross-check file names with NAME section
#
# for a given skopeo-foo.1.md, the NAME should be 'skopeo-foo'
for md in *.1.md;do
# Read the first line after '## NAME'
name=$(egrep -A1 '^## NAME' $md|tail -1|awk '{print $1}' | tr -d \\\\)
expect=$(basename $md .1.md)
if [ "$name" != "$expect" ]; then
echo
printf "Inconsistent program NAME in %s:\n" $md
printf " NAME= %s (expected: %s)\n" $name $expect
rc=1
fi
done
# Pass 2: compare descriptions.
#
# Make sure the descriptive text in skopeo-foo.1.md matches the one
# in the table in skopeo.1.md.
for md in $(ls -1 *-*.1.md);do
desc=$(egrep -A1 '^## NAME' $md|tail -1|sed -E -e 's/^skopeo[^[:space:]]+ - //')
# Find the descriptive text in the main skopeo man page.
parent=skopeo.1.md
parent_desc=$(grep $md $parent | awk -F'|' '{print $3}' | sed -E -e 's/^[[:space:]]+//' -e 's/[[:space:]]+$//')
if [ "$desc" != "$parent_desc" ]; then
echo
printf "Inconsistent subcommand descriptions:\n"
printf " %-32s = '%s'\n" $md "$desc"
printf " %-32s = '%s'\n" $parent "$parent_desc"
printf "Please ensure that the NAME section of $md\n"
printf "matches the subcommand description in $parent\n"
rc=1
fi
done
# Helper function: compares man page synopsis vs --help usage message
function compare_usage() {
local cmd="$1"
local from_man="$2"
# Run 'cmd --help', grab the line immediately after 'Usage:'
local help_output=$(../bin/$cmd --help)
local from_help=$(echo "$help_output" | grep -A1 '^Usage:' | tail -1)
# strip off command name from both
from_man=$(sed -E -e "s/\*\*$cmd\*\*[[:space:]]*//" <<<"$from_man")
from_help=$(sed -E -e "s/^[[:space:]]*$cmd[[:space:]]*//" <<<"$from_help")
# man page lists 'foo [*options*]', help msg shows 'foo [command options]'.
# Make sure if one has it, the other does too.
if expr "$from_man" : "\[\*options\*\]" >/dev/null; then
if expr "$from_help" : "\[command options\]" >/dev/null; then
:
else
echo "WARNING: $cmd: man page shows '[*options*]', help does not show [command options]"
rc=1
fi
elif expr "$from_help" : "\[command options\]" >/dev/null; then
echo "WARNING: $cmd: --help shows [command options], man page does not show [*options*]"
rc=1
fi
# Strip off options and flags; start comparing arguments
from_man=$(sed -E -e 's/^\[\*options\*\][[:space:]]*//' <<<"$from_man")
from_help=$(sed -E -e 's/^\[command options\][[:space:]]*//' <<<"$from_help")
# Constant strings in man page are '**foo**', in --help are 'foo'.
from_man=$(sed -E -e 's/\*\*([^*]+)\*\*/\1/g' <<<"$from_man")
# Args in man page are '_foo_', in --help are 'FOO'. Convert all to
# UPCASE simply because it stands out better to the eye.
from_man=$(sed -E -e 's/_([a-z-]+)_/\U\1/g' <<<"$from_man")
# Compare man-page and --help usage strings. Skip 'skopeo' itself,
# because the man page includes '[global options]' which we don't grok.
if [[ "$from_man" != "$from_help" && "$cmd" != "skopeo" ]]; then
printf "%-25s man='%s' help='%s'\n" "$cmd:" "$from_man" "$from_help"
rc=1
fi
}
# Pass 3: compare synopses.
#
# Make sure the SYNOPSIS line in skopeo-foo.1.md reads '**skopeo foo** ...'
for md in *.1.md;do
synopsis=$(egrep -A1 '^#* SYNOPSIS' $md|tail -1)
# Command name must be bracketed by double asterisks; options and
# arguments are bracketed by single ones.
# E.g. '**skopeo copy** [*options*] _..._'
# Get the command name, and confirm that it matches the md file name.
cmd=$(echo "$synopsis" | sed -E -e 's/^\*\*([^*]+)\*\*.*/\1/' | tr -d \*)
# Use sed, not tr, so we only replace the first dash: we want
# skopeo-list-tags -> "skopeo list-tags", not "skopeo list tags"
md_nodash=$(basename "$md" .1.md | sed -e 's/-/ /')
if [ "$cmd" != "$md_nodash" ]; then
echo
printf "Inconsistent program name in SYNOPSIS in %s:\n" $md
printf " SYNOPSIS = %s (expected: '%s')\n" "$cmd" "$md_nodash"
rc=1
fi
# The convention is to use UPPER CASE in 'skopeo foo --help',
# but *lower case bracketed by asterisks* in the man page
if expr "$synopsis" : ".*[A-Z]" >/dev/null; then
echo
printf "Inconsistent capitalization in SYNOPSIS in %s\n" $md
printf " '%s' should not contain upper-case characters\n" "$synopsis"
rc=1
fi
# (for debugging, and getting a sense of standard conventions)
#printf " %-32s ------ '%s'\n" $md "$synopsis"
# If bin/skopeo is available, run "cmd --help" and compare Usage
# messages. This is complicated, so do it in a helper function.
compare_usage "$md_nodash" "$synopsis"
done
exit $rc

17
hack/travis_osx.sh Executable file
View File

@@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -e
export GOPATH=$(pwd)/_gopath
export PATH=$GOPATH/bin:$PATH
_containers="${GOPATH}/src/github.com/containers"
mkdir -vp ${_containers}
ln -vsf $(pwd) ${_containers}/skopeo
go version
GO111MODULE=off go get -u github.com/cpuguy83/go-md2man golang.org/x/lint/golint
cd ${_containers}/skopeo
make validate-local test-unit-local binary-local
sudo make install
skopeo -v

View File

@@ -1,277 +0,0 @@
#!/usr/bin/perl
#
# xref-helpmsgs-manpages - cross-reference --help options against man pages
#
package LibPod::CI::XrefHelpmsgsManpages;
use v5.14;
use utf8;
use strict;
use warnings;
(our $ME = $0) =~ s|.*/||;
our $VERSION = '0.1';
# For debugging, show data structures using DumpTree($var)
#use Data::TreeDumper; $Data::TreeDumper::Displayaddress = 0;
# unbuffer output
$| = 1;
###############################################################################
# BEGIN user-customizable section
# Path to skopeo executable
my $Default_Skopeo = './bin/skopeo';
my $SKOPEO = $ENV{SKOPEO} || $Default_Skopeo;
# Path to all doc files (markdown)
my $Docs_Path = 'docs';
# Global error count
my $Errs = 0;
# END user-customizable section
###############################################################################
###############################################################################
# BEGIN boilerplate args checking, usage messages
sub usage {
print <<"END_USAGE";
Usage: $ME [OPTIONS]
$ME recursively runs 'skopeo --help' against
all subcommands; and recursively reads skopeo-*.1.md files
in $Docs_Path, then cross-references that each --help
option is listed in the appropriate man page and vice-versa.
$ME invokes '\$SKOPEO' (default: $Default_Skopeo).
Exit status is zero if no inconsistencies found, one otherwise
OPTIONS:
-v, --verbose show verbose progress indicators
-n, --dry-run make no actual changes
--help display this message
--version display program name and version
END_USAGE
exit;
}
# Command-line options. Note that this operates directly on @ARGV !
our $debug = 0;
our $verbose = 0;
sub handle_opts {
use Getopt::Long;
GetOptions(
'debug!' => \$debug,
'verbose|v' => \$verbose,
help => \&usage,
version => sub { print "$ME version $VERSION\n"; exit 0 },
) or die "Try `$ME --help' for help\n";
}
# END boilerplate args checking, usage messages
###############################################################################
############################## CODE BEGINS HERE ###############################
# The term is "modulino".
__PACKAGE__->main() unless caller();
# Main code.
sub main {
# Note that we operate directly on @ARGV, not on function parameters.
# This is deliberate: it's because Getopt::Long only operates on @ARGV
# and there's no clean way to make it use @_.
handle_opts(); # will set package globals
# Fetch command-line arguments. Barf if too many.
die "$ME: Too many arguments; try $ME --help\n" if @ARGV;
my $help = skopeo_help();
my $man = skopeo_man('skopeo');
xref_by_help($help, $man);
xref_by_man($help, $man);
exit !!$Errs;
}
###############################################################################
# BEGIN cross-referencing
##################
# xref_by_help # Find keys in '--help' but not in man
##################
sub xref_by_help {
my ($help, $man, @subcommand) = @_;
for my $k (sort keys %$help) {
if (exists $man->{$k}) {
if (ref $help->{$k}) {
xref_by_help($help->{$k}, $man->{$k}, @subcommand, $k);
}
# Otherwise, non-ref is leaf node such as a --option
}
else {
my $man = $man->{_path} || 'man';
warn "$ME: skopeo @subcommand --help lists $k, but $k not in $man\n";
++$Errs;
}
}
}
#################
# xref_by_man # Find keys in man pages but not in --help
#################
#
# In an ideal world we could share the functionality in one function; but
# there are just too many special cases in man pages.
#
sub xref_by_man {
my ($help, $man, @subcommand) = @_;
# FIXME: this generates way too much output
for my $k (grep { $_ ne '_path' } sort keys %$man) {
if (exists $help->{$k}) {
if (ref $man->{$k}) {
xref_by_man($help->{$k}, $man->{$k}, @subcommand, $k);
}
}
elsif ($k ne '--help' && $k ne '-h') {
my $man = $man->{_path} || 'man';
warn "$ME: skopeo @subcommand: $k in $man, but not --help\n";
++$Errs;
}
}
}
# END cross-referencing
###############################################################################
# BEGIN data gathering
#################
# skopeo_help # Parse output of 'skopeo [subcommand] --help'
#################
sub skopeo_help {
my %help;
open my $fh, '-|', $SKOPEO, @_, '--help'
or die "$ME: Cannot fork: $!\n";
my $section = '';
while (my $line = <$fh>) {
# Cobra is blessedly consistent in its output:
# Usage: ...
# Available Commands:
# ....
# Options:
# ....
#
# Start by identifying the section we're in...
if ($line =~ /^Available\s+(Commands):/) {
$section = lc $1;
}
elsif ($line =~ /^(Flags):/) {
$section = lc $1;
}
# ...then track commands and options. For subcommands, recurse.
elsif ($section eq 'commands') {
if ($line =~ /^\s{1,4}(\S+)\s/) {
my $subcommand = $1;
print "> skopeo @_ $subcommand\n" if $debug;
$help{$subcommand} = skopeo_help(@_, $subcommand)
unless $subcommand eq 'help'; # 'help' not in man
}
}
elsif ($section eq 'flags') {
# Handle '--foo' or '-f, --foo'
if ($line =~ /^\s{1,10}(--\S+)\s/) {
print "> skopeo @_ $1\n" if $debug;
$help{$1} = 1;
}
elsif ($line =~ /^\s{1,10}(-\S),\s+(--\S+)\s/) {
print "> skopeo @_ $1, $2\n" if $debug;
$help{$1} = $help{$2} = 1;
}
}
}
close $fh
or die "$ME: Error running 'skopeo @_ --help'\n";
return \%help;
}
################
# skopeo_man # Parse contents of skopeo-*.1.md
################
sub skopeo_man {
my $command = shift;
my $manpath = "$Docs_Path/$command.1.md";
print "** $manpath \n" if $debug;
my %man = (_path => $manpath);
open my $fh, '<', $manpath
or die "$ME: Cannot read $manpath: $!\n";
my $section = '';
my @most_recent_flags;
my $previous_subcmd = '';
while (my $line = <$fh>) {
chomp $line;
next unless $line; # skip empty lines
# .md files designate sections with leading double hash
if ($line =~ /^##\s*OPTIONS/) {
$section = 'flags';
}
elsif ($line =~ /^\#\#\s+(SUB)?COMMANDS/) {
$section = 'commands';
}
elsif ($line =~ /^\#\#[^#]/) {
$section = '';
}
# This will be a table containing subcommand names, links to man pages.
elsif ($section eq 'commands') {
# In skopeo.1.md
if ($line =~ /^\|\s*\[skopeo-(\S+?)\(\d\)\]/) {
# $1 will be changed by recursion _*BEFORE*_ left-hand assignment
my $subcmd = $1;
$man{$subcmd} = skopeo_man("skopeo-$1");
}
}
# Options should always be of the form '**-f**' or '**\-\-flag**',
# possibly separated by comma-space.
elsif ($section eq 'flags') {
# If option has long and short form, long must come first.
# This is a while-loop because there may be multiple long
# option names (not in skopeo ATM, but leave the possibility open)
while ($line =~ s/^\*\*(--[a-z0-9.-]+)\*\*(=\*[a-zA-Z0-9-]+\*)?(,\s+)?//g) {
$man{$1} = 1;
}
# Short form
if ($line =~ s/^\*\*(-[a-zA-Z0-9.])\*\*(=\*[a-zA-Z0-9-]+\*)?//g) {
$man{$1} = 1;
}
}
}
close $fh;
return \%man;
}
# END data gathering
###############################################################################
1;

View File

@@ -1,123 +1,76 @@
# Installing from packages
## Distribution Packages
`skopeo` may already be packaged in your distribution.
### Fedora
`skopeo` may already be packaged in your distribution, for example on
RHEL/CentOS ≥ 8 or Fedora you can install it using:
```sh
sudo dnf -y install skopeo
$ sudo dnf install skopeo
```
### RHEL/CentOS ≥ 8 and CentOS Stream
on RHEL/CentOS ≤ 7.x:
```sh
sudo dnf -y install skopeo
$ sudo yum install skopeo
```
Newer Skopeo releases may be available on the repositories provided by the
Kubic project. Beware, these may not be suitable for production environments.
on CentOS 8:
for openSUSE:
```sh
sudo dnf -y module disable container-tools
sudo dnf -y install 'dnf-command(copr)'
sudo dnf -y copr enable rhcontainerbot/container-selinux
sudo curl -L -o /etc/yum.repos.d/devel:kubic:libcontainers:stable.repo https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_8/devel:kubic:libcontainers:stable.repo
sudo dnf -y install skopeo
$ sudo zypper install skopeo
```
on CentOS 8 Stream:
on alpine:
```sh
sudo dnf -y module disable container-tools
sudo dnf -y install 'dnf-command(copr)'
sudo dnf -y copr enable rhcontainerbot/container-selinux
sudo curl -L -o /etc/yum.repos.d/devel:kubic:libcontainers:stable.repo https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_8_Stream/devel:kubic:libcontainers:stable.repo
sudo dnf -y install skopeo
$ sudo apk add skopeo
```
### RHEL/CentOS ≤ 7.x
on macOS:
```sh
sudo yum -y install skopeo
$ brew install skopeo
```
### openSUSE
Debian (10 and newer including Raspbian) and Ubuntu (18.04 and newer): Packages
are available via the [Kubic][0] project repositories:
```sh
sudo zypper install skopeo
```
### Alpine
```sh
sudo apk add skopeo
```
### macOS
```sh
brew install skopeo
```
### Nix / NixOS
```sh
$ nix-env -i skopeo
```
### Debian
The skopeo package is available in
the [Bullseye (testing) branch](https://packages.debian.org/bullseye/skopeo), which
will be the next stable release (Debian 11) as well as Debian Unstable/Sid.
[0]: https://build.opensuse.org/project/show/devel:kubic:libcontainers:stable
```bash
# Debian Testing/Bullseye or Unstable/Sid
sudo apt-get update
sudo apt-get -y install skopeo
# Debian Unstable/Sid:
$ echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_Unstable/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
$ wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/Debian_Unstable/Release.key -O- | sudo apt-key add -
```
### Raspberry Pi OS arm64 (beta)
Raspberry Pi OS uses the standard Debian's repositories,
so it is fully compatible with Debian's arm64 repository.
You can simply follow the [steps for Debian](#debian) to install Skopeo.
### Ubuntu
The skopeo package is available in the official repositories for Ubuntu 20.10
and newer.
```bash
# Ubuntu 20.10 and newer
sudo apt-get -y update
sudo apt-get -y install skopeo
# Debian Testing:
$ echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_Testing/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
$ wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/Debian_Testing/Release.key -O- | sudo apt-key add -
```
If you would prefer newer (though not as well-tested) packages,
the [Kubic project](https://build.opensuse.org/package/show/devel:kubic:libcontainers:stable/skopeo)
provides packages for active Ubuntu releases 20.04 and newer (it should also work with direct derivatives like Pop!\_OS).
Checkout the [Kubic project page](https://build.opensuse.org/package/show/devel:kubic:libcontainers:stable/skopeo)
for a list of supported Ubuntu version and
architecture combinations. **NOTE:** The command `sudo apt-get -y upgrade`
maybe required in some cases if Skopeo cannot be installed without it.
The build sources for the Kubic packages can be found [here](https://gitlab.com/rhcontainerbot/skopeo/-/tree/debian/debian).
CAUTION: On Ubuntu 20.10 and newer, we highly recommend you use Buildah, Podman and Skopeo ONLY from EITHER the Kubic repo
OR the official Ubuntu repos. Mixing and matching may lead to unpredictable situations including installation conflicts.
```bash
. /etc/os-release
echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add -
sudo apt-get update
sudo apt-get -y upgrade
sudo apt-get -y install skopeo
# Debian 10:
$ echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
$ wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/Debian_10/Release.key -O- | sudo apt-key add -
```
```bash
# Raspbian 10:
$ echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Raspbian_10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
$ wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/Raspbian_10/Release.key -O- | sudo apt-key add -
```
```bash
# Ubuntu (18.04, 19.04 and 19.10):
$ . /etc/os-release
$ sudo sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/x${NAME}_${VERSION_ID}/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list"
$ wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/x${NAME}_${VERSION_ID}/Release.key -O- | sudo apt-key add -
```
```bash
$ sudo apt-get update -qq
$ sudo apt-get install skopeo
```
Otherwise, read on for building and installing it from source:
@@ -126,8 +79,6 @@ To build the `skopeo` binary you need at least Go 1.12.
There are two ways to build skopeo: in a container, or locally without a
container. Choose the one which better matches your needs and environment.
## Building from Source
### Building without a container
Building without a container requires a bit more manual work and setup in your
@@ -141,37 +92,48 @@ Install the necessary dependencies:
```bash
# Fedora:
sudo dnf install gpgme-devel libassuan-devel btrfs-progs-devel device-mapper-devel
$ sudo dnf install gpgme-devel libassuan-devel btrfs-progs-devel device-mapper-devel
```
```bash
# Ubuntu (`libbtrfs-dev` requires Ubuntu 18.10 and above):
sudo apt install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config
$ sudo apt install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev
```
```bash
# macOS:
brew install gpgme
$ brew install gpgme
```
```bash
# openSUSE:
sudo zypper install libgpgme-devel device-mapper-devel libbtrfs-devel glib2-devel
$ sudo zypper install libgpgme-devel device-mapper-devel libbtrfs-devel glib2-devel
```
Make sure to clone this repository in your `GOPATH` - otherwise compilation fails.
```bash
git clone https://github.com/containers/skopeo $GOPATH/src/github.com/containers/skopeo
cd $GOPATH/src/github.com/containers/skopeo && make bin/skopeo
$ git clone https://github.com/containers/skopeo $GOPATH/src/github.com/containers/skopeo
$ cd $GOPATH/src/github.com/containers/skopeo && make binary-local
```
By default the `make` command (make all) will build bin/skopeo and the documentation locally.
### Building in a container
Building of documentation requires `go-md2man`. On systems that do not have this tool, the
document generation can be skipped by passing `DISABLE_DOCS=1`:
Building in a container is simpler, but more restrictive:
- It requires the `podman` command and the ability to run Linux containers
- The created executable is a Linux executable, and depends on dynamic libraries
which may only be available only in a container of a similar Linux
distribution.
```bash
$ make binary # Or (make all) to also build documentation, see below.
```
DISABLE_DOCS=1 make
To build a pure-Go static binary (disables devicemapper, btrfs, and gpgme):
```bash
$ make binary-static DISABLE_CGO=1
```
### Building documentation
@@ -180,36 +142,18 @@ To build the manual you will need go-md2man.
```bash
# Debian:
sudo apt-get install go-md2man
$ sudo apt-get install go-md2man
```
```
# Fedora:
sudo dnf install go-md2man
```
```
# MacOS:
brew install go-md2man
$ sudo dnf install go-md2man
```
Then
```bash
make docs
```
### Building in a container
Building in a container is simpler, but more restrictive:
- It requires the `podman` command and the ability to run Linux containers.
- The created executable is a Linux executable, and depends on dynamic libraries
which may only be available only in a container of a similar Linux
distribution.
```bash
$ make binary
$ make docs
```
### Installation
@@ -217,5 +161,5 @@ $ make binary
Finally, after the binary and documentation is built:
```bash
sudo make install
$ sudo make install
```

View File

@@ -101,8 +101,8 @@ func (s *SkopeoSuite) TestCopyWithLocalAuth(c *check.C) {
assertSkopeoSucceeds(c, wanted, "login", "--tls-verify=false", "--username="+s.regV2WithAuth.username, "--password="+s.regV2WithAuth.password, s.regV2WithAuth.url)
// copy to private registry using local authentication
imageName := fmt.Sprintf("docker://%s/busybox:mine", s.regV2WithAuth.url)
assertSkopeoSucceeds(c, "", "copy", "--dest-tls-verify=false", testFQIN+":latest", imageName)
// inspect from private registry
assertSkopeoSucceeds(c, "", "copy", "--dest-tls-verify=false", "docker://docker.io/library/busybox:latest", imageName)
// inspec from private registry
assertSkopeoSucceeds(c, "", "inspect", "--tls-verify=false", imageName)
// logout from the registry
wanted = fmt.Sprintf("^Removed login credentials for %s\n$", s.regV2WithAuth.url)

View File

@@ -31,7 +31,6 @@ const (
v2DockerRegistryURL = "localhost:5555" // Update also policy.json
v2s1DockerRegistryURL = "localhost:5556"
knownWindowsOnlyImage = "docker://mcr.microsoft.com/windows/nanoserver:1909"
knownListImage = "docker://registry.fedoraproject.org/fedora-minimal" // could have either ":latest" or "@sha256:..." appended
)
type CopySuite struct {
@@ -100,14 +99,14 @@ func (s *CopySuite) TestCopyWithManifestList(c *check.C) {
dir, err := ioutil.TempDir("", "copy-manifest-list")
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir)
assertSkopeoSucceeds(c, "", "copy", knownListImage, "dir:"+dir)
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox:latest", "dir:"+dir)
}
func (s *CopySuite) TestCopyAllWithManifestList(c *check.C) {
dir, err := ioutil.TempDir("", "copy-all-manifest-list")
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir)
assertSkopeoSucceeds(c, "", "copy", "--all", knownListImage, "dir:"+dir)
assertSkopeoSucceeds(c, "", "copy", "--all", "docker://estesp/busybox:latest", "dir:"+dir)
}
func (s *CopySuite) TestCopyAllWithManifestListRoundTrip(c *check.C) {
@@ -123,7 +122,7 @@ func (s *CopySuite) TestCopyAllWithManifestListRoundTrip(c *check.C) {
dir2, err := ioutil.TempDir("", "copy-all-manifest-list-dir")
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir2)
assertSkopeoSucceeds(c, "", "copy", "--all", knownListImage, "oci:"+oci1)
assertSkopeoSucceeds(c, "", "copy", "--all", "docker://estesp/busybox:latest", "oci:"+oci1)
assertSkopeoSucceeds(c, "", "copy", "--all", "oci:"+oci1, "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", "--all", "dir:"+dir1, "oci:"+oci2)
assertSkopeoSucceeds(c, "", "copy", "--all", "oci:"+oci2, "dir:"+dir2)
@@ -145,9 +144,9 @@ func (s *CopySuite) TestCopyAllWithManifestListConverge(c *check.C) {
dir2, err := ioutil.TempDir("", "copy-all-manifest-list-dir")
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir2)
assertSkopeoSucceeds(c, "", "copy", "--all", knownListImage, "oci:"+oci1)
assertSkopeoSucceeds(c, "", "copy", "--all", "docker://estesp/busybox:latest", "oci:"+oci1)
assertSkopeoSucceeds(c, "", "copy", "--all", "oci:"+oci1, "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", "--all", "--format", "oci", knownListImage, "dir:"+dir2)
assertSkopeoSucceeds(c, "", "copy", "--all", "--format", "oci", "docker://estesp/busybox:latest", "dir:"+dir2)
assertSkopeoSucceeds(c, "", "copy", "--all", "dir:"+dir2, "oci:"+oci2)
assertDirImagesAreEqual(c, dir1, dir2)
out := combinedOutputOfCommand(c, "diff", "-urN", oci1, oci2)
@@ -167,9 +166,9 @@ func (s *CopySuite) TestCopyWithManifestListConverge(c *check.C) {
dir2, err := ioutil.TempDir("", "copy-all-manifest-list-dir")
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir2)
assertSkopeoSucceeds(c, "", "copy", knownListImage, "oci:"+oci1)
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox:latest", "oci:"+oci1)
assertSkopeoSucceeds(c, "", "copy", "--all", "oci:"+oci1, "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", "--format", "oci", knownListImage, "dir:"+dir2)
assertSkopeoSucceeds(c, "", "copy", "--format", "oci", "docker://estesp/busybox:latest", "dir:"+dir2)
assertSkopeoSucceeds(c, "", "copy", "--all", "dir:"+dir2, "oci:"+oci2)
assertDirImagesAreEqual(c, dir1, dir2)
out := combinedOutputOfCommand(c, "diff", "-urN", oci1, oci2)
@@ -181,7 +180,7 @@ func (s *CopySuite) TestCopyAllWithManifestListStorageFails(c *check.C) {
c.Assert(err, check.IsNil)
defer os.RemoveAll(storage)
storage = fmt.Sprintf("[vfs@%s/root+%s/runroot]", storage, storage)
assertSkopeoFails(c, `.*destination transport .* does not support copying multiple images as a group.*`, "copy", "--all", knownListImage, "containers-storage:"+storage+"test")
assertSkopeoFails(c, `.*destination transport .* does not support copying multiple images as a group.*`, "copy", "--all", "docker://estesp/busybox:latest", "containers-storage:"+storage+"test")
}
func (s *CopySuite) TestCopyWithManifestListStorage(c *check.C) {
@@ -195,8 +194,8 @@ func (s *CopySuite) TestCopyWithManifestListStorage(c *check.C) {
dir2, err := ioutil.TempDir("", "copy-manifest-list-storage-dir")
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir2)
assertSkopeoSucceeds(c, "", "copy", knownListImage, "containers-storage:"+storage+"test")
assertSkopeoSucceeds(c, "", "copy", knownListImage, "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox:latest", "containers-storage:"+storage+"test")
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox:latest", "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", "containers-storage:"+storage+"test", "dir:"+dir2)
runDecompressDirs(c, "", dir1, dir2)
assertDirImagesAreEqual(c, dir1, dir2)
@@ -213,9 +212,9 @@ func (s *CopySuite) TestCopyWithManifestListStorageMultiple(c *check.C) {
dir2, err := ioutil.TempDir("", "copy-manifest-list-storage-multiple-dir")
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir2)
assertSkopeoSucceeds(c, "", "--override-arch", "amd64", "copy", knownListImage, "containers-storage:"+storage+"test")
assertSkopeoSucceeds(c, "", "--override-arch", "arm64", "copy", knownListImage, "containers-storage:"+storage+"test")
assertSkopeoSucceeds(c, "", "--override-arch", "arm64", "copy", knownListImage, "dir:"+dir1)
assertSkopeoSucceeds(c, "", "--override-arch", "amd64", "copy", "docker://estesp/busybox:latest", "containers-storage:"+storage+"test")
assertSkopeoSucceeds(c, "", "--override-arch", "arm64", "copy", "docker://estesp/busybox:latest", "containers-storage:"+storage+"test")
assertSkopeoSucceeds(c, "", "--override-arch", "arm64", "copy", "docker://estesp/busybox:latest", "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", "containers-storage:"+storage+"test", "dir:"+dir2)
runDecompressDirs(c, "", dir1, dir2)
assertDirImagesAreEqual(c, dir1, dir2)
@@ -234,33 +233,18 @@ func (s *CopySuite) TestCopyWithManifestListDigest(c *check.C) {
oci2, err := ioutil.TempDir("", "copy-manifest-list-digest-oci")
c.Assert(err, check.IsNil)
defer os.RemoveAll(oci2)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", knownListImage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", "docker://estesp/busybox:latest")
manifestDigest, err := manifest.Digest([]byte(m))
c.Assert(err, check.IsNil)
digest := manifestDigest.String()
assertSkopeoSucceeds(c, "", "copy", knownListImage+"@"+digest, "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", "--all", knownListImage+"@"+digest, "dir:"+dir2)
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox@"+digest, "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", "--all", "docker://estesp/busybox@"+digest, "dir:"+dir2)
assertSkopeoSucceeds(c, "", "copy", "dir:"+dir1, "oci:"+oci1)
assertSkopeoSucceeds(c, "", "copy", "dir:"+dir2, "oci:"+oci2)
out := combinedOutputOfCommand(c, "diff", "-urN", oci1, oci2)
c.Assert(out, check.Equals, "")
}
func (s *CopySuite) TestCopyWithDigestfileOutput(c *check.C) {
tempdir, err := ioutil.TempDir("", "tempdir")
c.Assert(err, check.IsNil)
defer os.RemoveAll(tempdir)
dir1, err := ioutil.TempDir("", "copy-manifest-list-digest-dir")
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir1)
digestOutPath := filepath.Join(tempdir, "digest.txt")
assertSkopeoSucceeds(c, "", "copy", "--digestfile="+digestOutPath, knownListImage, "dir:"+dir1)
readDigest, err := ioutil.ReadFile(digestOutPath)
c.Assert(err, check.IsNil)
_, err = digest.Parse(string(readDigest))
c.Assert(err, check.IsNil)
}
func (s *CopySuite) TestCopyWithManifestListStorageDigest(c *check.C) {
storage, err := ioutil.TempDir("", "copy-manifest-list-storage-digest")
c.Assert(err, check.IsNil)
@@ -272,13 +256,13 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigest(c *check.C) {
dir2, err := ioutil.TempDir("", "copy-manifest-list-storage-digest-dir")
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir2)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", knownListImage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", "docker://estesp/busybox:latest")
manifestDigest, err := manifest.Digest([]byte(m))
c.Assert(err, check.IsNil)
digest := manifestDigest.String()
assertSkopeoSucceeds(c, "", "copy", knownListImage+"@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "copy", "containers-storage:"+storage+"test@"+digest, "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", knownListImage+"@"+digest, "dir:"+dir2)
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox@"+digest, "dir:"+dir2)
runDecompressDirs(c, "", dir1, dir2)
assertDirImagesAreEqual(c, dir1, dir2)
}
@@ -294,13 +278,13 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigestMultipleArches(c *check
dir2, err := ioutil.TempDir("", "copy-manifest-list-storage-digest-dir")
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir2)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", knownListImage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", "docker://estesp/busybox:latest")
manifestDigest, err := manifest.Digest([]byte(m))
c.Assert(err, check.IsNil)
digest := manifestDigest.String()
assertSkopeoSucceeds(c, "", "copy", knownListImage+"@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "copy", "containers-storage:"+storage+"test@"+digest, "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", knownListImage+"@"+digest, "dir:"+dir2)
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox@"+digest, "dir:"+dir2)
runDecompressDirs(c, "", dir1, dir2)
assertDirImagesAreEqual(c, dir1, dir2)
}
@@ -310,16 +294,16 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigestMultipleArchesBothUseLi
c.Assert(err, check.IsNil)
defer os.RemoveAll(storage)
storage = fmt.Sprintf("[vfs@%s/root+%s/runroot]", storage, storage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", knownListImage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", "docker://estesp/busybox:latest")
manifestDigest, err := manifest.Digest([]byte(m))
c.Assert(err, check.IsNil)
digest := manifestDigest.String()
_, err = manifest.ListFromBlob([]byte(m), manifest.GuessMIMEType([]byte(m)))
c.Assert(err, check.IsNil)
assertSkopeoSucceeds(c, "", "--override-arch=amd64", "copy", knownListImage+"@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "--override-arch=arm64", "copy", knownListImage+"@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*reading manifest for image instance.*does not exist.*`, "--override-arch=amd64", "inspect", "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*reading manifest for image instance.*does not exist.*`, "--override-arch=amd64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "--override-arch=amd64", "copy", "docker://estesp/busybox@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "--override-arch=arm64", "copy", "docker://estesp/busybox@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*error reading manifest for image instance.*does not exist.*`, "--override-arch=amd64", "inspect", "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*error reading manifest for image instance.*does not exist.*`, "--override-arch=amd64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
i2 := combinedOutputOfCommand(c, skopeoBinary, "--override-arch=arm64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
var image2 imgspecv1.Image
err = json.Unmarshal([]byte(i2), &image2)
@@ -332,7 +316,7 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigestMultipleArchesFirstUses
c.Assert(err, check.IsNil)
defer os.RemoveAll(storage)
storage = fmt.Sprintf("[vfs@%s/root+%s/runroot]", storage, storage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", knownListImage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", "docker://estesp/busybox:latest")
manifestDigest, err := manifest.Digest([]byte(m))
c.Assert(err, check.IsNil)
digest := manifestDigest.String()
@@ -342,8 +326,8 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigestMultipleArchesFirstUses
c.Assert(err, check.IsNil)
arm64Instance, err := list.ChooseInstance(&types.SystemContext{ArchitectureChoice: "arm64"})
c.Assert(err, check.IsNil)
assertSkopeoSucceeds(c, "", "--override-arch=amd64", "copy", knownListImage+"@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "--override-arch=arm64", "copy", knownListImage+"@"+arm64Instance.String(), "containers-storage:"+storage+"test@"+arm64Instance.String())
assertSkopeoSucceeds(c, "", "--override-arch=amd64", "copy", "docker://estesp/busybox@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "--override-arch=arm64", "copy", "docker://estesp/busybox@"+arm64Instance.String(), "containers-storage:"+storage+"test@"+arm64Instance.String())
i1 := combinedOutputOfCommand(c, skopeoBinary, "--override-arch=amd64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
var image1 imgspecv1.Image
err = json.Unmarshal([]byte(i1), &image1)
@@ -354,8 +338,8 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigestMultipleArchesFirstUses
err = json.Unmarshal([]byte(i2), &image2)
c.Assert(err, check.IsNil)
c.Assert(image2.Architecture, check.Equals, "amd64")
assertSkopeoFails(c, `.*reading manifest for image instance.*does not exist.*`, "--override-arch=arm64", "inspect", "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*reading manifest for image instance.*does not exist.*`, "--override-arch=arm64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*error reading manifest for image instance.*does not exist.*`, "--override-arch=arm64", "inspect", "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*error reading manifest for image instance.*does not exist.*`, "--override-arch=arm64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
i3 := combinedOutputOfCommand(c, skopeoBinary, "--override-arch=arm64", "inspect", "--config", "containers-storage:"+storage+"test@"+arm64Instance.String())
var image3 imgspecv1.Image
err = json.Unmarshal([]byte(i3), &image3)
@@ -368,7 +352,7 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigestMultipleArchesSecondUse
c.Assert(err, check.IsNil)
defer os.RemoveAll(storage)
storage = fmt.Sprintf("[vfs@%s/root+%s/runroot]", storage, storage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", knownListImage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", "docker://estesp/busybox:latest")
manifestDigest, err := manifest.Digest([]byte(m))
c.Assert(err, check.IsNil)
digest := manifestDigest.String()
@@ -378,15 +362,15 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigestMultipleArchesSecondUse
c.Assert(err, check.IsNil)
arm64Instance, err := list.ChooseInstance(&types.SystemContext{ArchitectureChoice: "arm64"})
c.Assert(err, check.IsNil)
assertSkopeoSucceeds(c, "", "--override-arch=amd64", "copy", knownListImage+"@"+amd64Instance.String(), "containers-storage:"+storage+"test@"+amd64Instance.String())
assertSkopeoSucceeds(c, "", "--override-arch=arm64", "copy", knownListImage+"@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "--override-arch=amd64", "copy", "docker://estesp/busybox@"+amd64Instance.String(), "containers-storage:"+storage+"test@"+amd64Instance.String())
assertSkopeoSucceeds(c, "", "--override-arch=arm64", "copy", "docker://estesp/busybox@"+digest, "containers-storage:"+storage+"test@"+digest)
i1 := combinedOutputOfCommand(c, skopeoBinary, "--override-arch=amd64", "inspect", "--config", "containers-storage:"+storage+"test@"+amd64Instance.String())
var image1 imgspecv1.Image
err = json.Unmarshal([]byte(i1), &image1)
c.Assert(err, check.IsNil)
c.Assert(image1.Architecture, check.Equals, "amd64")
assertSkopeoFails(c, `.*reading manifest for image instance.*does not exist.*`, "--override-arch=amd64", "inspect", "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*reading manifest for image instance.*does not exist.*`, "--override-arch=amd64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*error reading manifest for image instance.*does not exist.*`, "--override-arch=amd64", "inspect", "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*error reading manifest for image instance.*does not exist.*`, "--override-arch=amd64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
i2 := combinedOutputOfCommand(c, skopeoBinary, "--override-arch=arm64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
var image2 imgspecv1.Image
err = json.Unmarshal([]byte(i2), &image2)
@@ -404,7 +388,7 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigestMultipleArchesThirdUses
c.Assert(err, check.IsNil)
defer os.RemoveAll(storage)
storage = fmt.Sprintf("[vfs@%s/root+%s/runroot]", storage, storage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", knownListImage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", "docker://estesp/busybox:latest")
manifestDigest, err := manifest.Digest([]byte(m))
c.Assert(err, check.IsNil)
digest := manifestDigest.String()
@@ -414,10 +398,10 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigestMultipleArchesThirdUses
c.Assert(err, check.IsNil)
arm64Instance, err := list.ChooseInstance(&types.SystemContext{ArchitectureChoice: "arm64"})
c.Assert(err, check.IsNil)
assertSkopeoSucceeds(c, "", "--override-arch=amd64", "copy", knownListImage+"@"+amd64Instance.String(), "containers-storage:"+storage+"test@"+amd64Instance.String())
assertSkopeoSucceeds(c, "", "--override-arch=amd64", "copy", knownListImage+"@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "--override-arch=arm64", "copy", knownListImage+"@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*reading manifest for image instance.*does not exist.*`, "--override-arch=amd64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "--override-arch=amd64", "copy", "docker://estesp/busybox@"+amd64Instance.String(), "containers-storage:"+storage+"test@"+amd64Instance.String())
assertSkopeoSucceeds(c, "", "--override-arch=amd64", "copy", "docker://estesp/busybox@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "--override-arch=arm64", "copy", "docker://estesp/busybox@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*error reading manifest for image instance.*does not exist.*`, "--override-arch=amd64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
i1 := combinedOutputOfCommand(c, skopeoBinary, "--override-arch=amd64", "inspect", "--config", "containers-storage:"+storage+"test@"+amd64Instance.String())
var image1 imgspecv1.Image
err = json.Unmarshal([]byte(i1), &image1)
@@ -440,7 +424,7 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigestMultipleArchesTagAndDig
c.Assert(err, check.IsNil)
defer os.RemoveAll(storage)
storage = fmt.Sprintf("[vfs@%s/root+%s/runroot]", storage, storage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", knownListImage)
m := combinedOutputOfCommand(c, skopeoBinary, "inspect", "--raw", "docker://estesp/busybox:latest")
manifestDigest, err := manifest.Digest([]byte(m))
c.Assert(err, check.IsNil)
digest := manifestDigest.String()
@@ -450,9 +434,9 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigestMultipleArchesTagAndDig
c.Assert(err, check.IsNil)
arm64Instance, err := list.ChooseInstance(&types.SystemContext{ArchitectureChoice: "arm64"})
c.Assert(err, check.IsNil)
assertSkopeoSucceeds(c, "", "--override-arch=amd64", "copy", knownListImage, "containers-storage:"+storage+"test:latest")
assertSkopeoSucceeds(c, "", "--override-arch=arm64", "copy", knownListImage+"@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*reading manifest for image instance.*does not exist.*`, "--override-arch=amd64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
assertSkopeoSucceeds(c, "", "--override-arch=amd64", "copy", "docker://estesp/busybox:latest", "containers-storage:"+storage+"test:latest")
assertSkopeoSucceeds(c, "", "--override-arch=arm64", "copy", "docker://estesp/busybox@"+digest, "containers-storage:"+storage+"test@"+digest)
assertSkopeoFails(c, `.*error reading manifest for image instance.*does not exist.*`, "--override-arch=amd64", "inspect", "--config", "containers-storage:"+storage+"test@"+digest)
i1 := combinedOutputOfCommand(c, skopeoBinary, "--override-arch=arm64", "inspect", "--config", "containers-storage:"+storage+"test:latest")
var image1 imgspecv1.Image
err = json.Unmarshal([]byte(i1), &image1)
@@ -480,16 +464,16 @@ func (s *CopySuite) TestCopyWithManifestListStorageDigestMultipleArchesTagAndDig
c.Assert(image5.Architecture, check.Equals, "arm64")
}
func (s *CopySuite) TestCopyFailsWhenImageOSDoesNotMatchRuntimeOS(c *check.C) {
storage, err := ioutil.TempDir("", "copy-fails-image-does-not-match-runtime")
func (s *CopySuite) TestCopyFailsWhenImageOSDoesntMatchRuntimeOS(c *check.C) {
storage, err := ioutil.TempDir("", "copy-fails-image-doesnt-match-runtime")
c.Assert(err, check.IsNil)
defer os.RemoveAll(storage)
storage = fmt.Sprintf("[vfs@%s/root+%s/runroot]", storage, storage)
assertSkopeoFails(c, `.*no image found in manifest list for architecture .*, variant .*, OS .*`, "copy", knownWindowsOnlyImage, "containers-storage:"+storage+"test")
}
func (s *CopySuite) TestCopySucceedsWhenImageDoesNotMatchRuntimeButWeOverride(c *check.C) {
storage, err := ioutil.TempDir("", "copy-succeeds-image-does-not-match-runtime-but-override")
func (s *CopySuite) TestCopySucceedsWhenImageDoesntMatchRuntimeButWeOverride(c *check.C) {
storage, err := ioutil.TempDir("", "copy-succeeds-image-doesnt-match-runtime-but-override")
c.Assert(err, check.IsNil)
defer os.RemoveAll(storage)
storage = fmt.Sprintf("[vfs@%s/root+%s/runroot]", storage, storage)
@@ -504,14 +488,14 @@ func (s *CopySuite) TestCopySimpleAtomicRegistry(c *check.C) {
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir2)
// FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection.
// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
// "pull": docker: → dir:
assertSkopeoSucceeds(c, "", "copy", testFQIN64, "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox:amd64", "dir:"+dir1)
// "push": dir: → atomic:
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--debug", "copy", "dir:"+dir1, "atomic:localhost:5000/myns/unsigned:unsigned")
// The result of pushing and pulling is an equivalent image, except for schema1 embedded names.
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "atomic:localhost:5000/myns/unsigned:unsigned", "dir:"+dir2)
assertSchema1DirImagesAreEqualExceptNames(c, dir1, "libpod/busybox:amd64", dir2, "myns/unsigned:unsigned")
assertSchema1DirImagesAreEqualExceptNames(c, dir1, "estesp/busybox:amd64", dir2, "myns/unsigned:unsigned")
}
// The most basic (skopeo copy) use:
@@ -525,30 +509,30 @@ func (s *CopySuite) TestCopySimple(c *check.C) {
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir2)
// FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection.
// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
// "pull": docker: → dir:
assertSkopeoSucceeds(c, "", "copy", "docker://k8s.gcr.io/pause", "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", "docker://busybox", "dir:"+dir1)
// "push": dir: → docker(v2s2):
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--debug", "copy", "dir:"+dir1, ourRegistry+"pause:unsigned")
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--debug", "copy", "dir:"+dir1, ourRegistry+"busybox:unsigned")
// The result of pushing and pulling is an unmodified image.
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", ourRegistry+"pause:unsigned", "dir:"+dir2)
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", ourRegistry+"busybox:unsigned", "dir:"+dir2)
out := combinedOutputOfCommand(c, "diff", "-urN", dir1, dir2)
c.Assert(out, check.Equals, "")
// docker v2s2 -> OCI image layout with image name
// ociDest will be created by oci: if it doesn't exist
// so don't create it here to exercise auto-creation
ociDest := "pause-latest-image"
ociImgName := "pause"
ociDest := "busybox-latest-image"
ociImgName := "busybox"
defer os.RemoveAll(ociDest)
assertSkopeoSucceeds(c, "", "copy", "docker://k8s.gcr.io/pause:latest", "oci:"+ociDest+":"+ociImgName)
assertSkopeoSucceeds(c, "", "copy", "docker://busybox:latest", "oci:"+ociDest+":"+ociImgName)
_, err = os.Stat(ociDest)
c.Assert(err, check.IsNil)
// docker v2s2 -> OCI image layout without image name
ociDest = "pause-latest-noimage"
ociDest = "busybox-latest-noimage"
defer os.RemoveAll(ociDest)
assertSkopeoSucceeds(c, "", "copy", "docker://k8s.gcr.io/pause:latest", "oci:"+ociDest)
assertSkopeoSucceeds(c, "", "copy", "docker://busybox:latest", "oci:"+ociDest)
_, err = os.Stat(ociDest)
c.Assert(err, check.IsNil)
}
@@ -568,7 +552,6 @@ func (s *CopySuite) TestCopyEncryption(c *check.C) {
c.Assert(err, check.IsNil)
defer os.RemoveAll(keysDir)
undecryptedImgDir, err := ioutil.TempDir("", "copy-5")
c.Assert(err, check.IsNil)
defer os.RemoveAll(undecryptedImgDir)
multiLayerImageDir, err := ioutil.TempDir("", "copy-6")
c.Assert(err, check.IsNil)
@@ -603,7 +586,7 @@ func (s *CopySuite) TestCopyEncryption(c *check.C) {
"oci:"+encryptedImgDir+":encrypted", "oci:"+decryptedImgDir+":decrypted")
// Copy a standard busybox image locally
assertSkopeoSucceeds(c, "", "copy", testFQIN+":1.30.1", "oci:"+originalImageDir+":latest")
assertSkopeoSucceeds(c, "", "copy", "docker://busybox:1.31.1", "oci:"+originalImageDir+":latest")
// Encrypt the image
assertSkopeoSucceeds(c, "", "copy", "--encryption-key",
@@ -634,7 +617,7 @@ func (s *CopySuite) TestCopyEncryption(c *check.C) {
matchLayerBlobBinaryType(c, decryptedImgDir+"/blobs/sha256", "application/x-gzip", 1)
// Copy a standard multi layer nginx image locally
assertSkopeoSucceeds(c, "", "copy", testFQINMultiLayer, "oci:"+multiLayerImageDir+":latest")
assertSkopeoSucceeds(c, "", "copy", "docker://nginx:1.17.8", "oci:"+multiLayerImageDir+":latest")
// Partially encrypt the image
assertSkopeoSucceeds(c, "", "copy", "--encryption-key", "jwe:"+keysDir+"/public.key",
@@ -643,7 +626,7 @@ func (s *CopySuite) TestCopyEncryption(c *check.C) {
// Since the image is partially encrypted we should find layers that aren't encrypted
matchLayerBlobBinaryType(c, partiallyEncryptedImgDir+"/blobs/sha256", "application/x-gzip", 2)
// Decrypt the partially encrypted image
// Decrypt the partically encrypted image
assertSkopeoSucceeds(c, "", "copy", "--decryption-key", keysDir+"/private.key",
"oci:"+partiallyEncryptedImgDir+":encrypted", "oci:"+partiallyDecryptedImgDir+":decrypted")
@@ -737,13 +720,13 @@ func (s *CopySuite) TestCopyStreaming(c *check.C) {
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir2)
// FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection.
// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
// streaming: docker: → atomic:
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--debug", "copy", testFQIN64, "atomic:localhost:5000/myns/unsigned:streaming")
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--debug", "copy", "docker://estesp/busybox:amd64", "atomic:localhost:5000/myns/unsigned:streaming")
// Compare (copies of) the original and the copy:
assertSkopeoSucceeds(c, "", "copy", testFQIN64, "dir:"+dir1)
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox:amd64", "dir:"+dir1)
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "atomic:localhost:5000/myns/unsigned:streaming", "dir:"+dir2)
assertSchema1DirImagesAreEqualExceptNames(c, dir1, "libpod/busybox:amd64", dir2, "myns/unsigned:streaming")
assertSchema1DirImagesAreEqualExceptNames(c, dir1, "estesp/busybox:amd64", dir2, "myns/unsigned:streaming")
// FIXME: Also check pushing to docker://
}
@@ -763,7 +746,7 @@ func (s *CopySuite) TestCopyOCIRoundTrip(c *check.C) {
defer os.RemoveAll(oci2)
// Docker -> OCI
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--debug", "copy", testFQIN, "oci:"+oci1+":latest")
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--debug", "copy", "docker://busybox", "oci:"+oci1+":latest")
// OCI -> Docker
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--debug", "copy", "oci:"+oci1+":latest", ourRegistry+"original/busybox:oci_copy")
// Docker -> OCI
@@ -814,16 +797,16 @@ func (s *CopySuite) TestCopySignatures(c *check.C) {
defer os.Remove(policy)
// type: reject
assertSkopeoFails(c, fmt.Sprintf(".*Source image rejected: Running image %s:latest is rejected by policy.*", testFQIN),
"--policy", policy, "copy", testFQIN+":latest", dirDest)
assertSkopeoFails(c, ".*Source image rejected: Running image docker://busybox:latest is rejected by policy.*",
"--policy", policy, "copy", "docker://busybox:latest", dirDest)
// type: insecureAcceptAnything
assertSkopeoSucceeds(c, "", "--policy", policy, "copy", "docker://quay.io/openshift/origin-hello-openshift", dirDest)
assertSkopeoSucceeds(c, "", "--policy", policy, "copy", "docker://openshift/hello-openshift", dirDest)
// type: signedBy
// Sign the images
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "--sign-by", "personal@example.com", testFQIN+":1.26", "atomic:localhost:5006/myns/personal:personal")
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "--sign-by", "official@example.com", testFQIN+":1.26.1", "atomic:localhost:5006/myns/official:official")
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "--sign-by", "personal@example.com", "docker://busybox:1.26", "atomic:localhost:5006/myns/personal:personal")
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "--sign-by", "official@example.com", "docker://busybox:1.26.1", "atomic:localhost:5006/myns/official:official")
// Verify that we can pull them
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--policy", policy, "copy", "atomic:localhost:5006/myns/personal:personal", dirDest)
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--policy", policy, "copy", "atomic:localhost:5006/myns/official:official", dirDest)
@@ -877,10 +860,10 @@ func (s *CopySuite) TestCopyDirSignatures(c *check.C) {
defer os.Remove(policy)
// Get some images.
assertSkopeoSucceeds(c, "", "copy", testFQIN+":armfh", topDirDest+"/dir1")
assertSkopeoSucceeds(c, "", "copy", testFQIN+":s390x", topDirDest+"/dir2")
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox:armfh", topDirDest+"/dir1")
assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox:s390x", topDirDest+"/dir2")
// Sign the images. By coping from a topDirDest/dirN, also test that non-/restricted paths
// Sign the images. By coping fom a topDirDest/dirN, also test that non-/restricted paths
// use the dir:"" default of insecureAcceptAnything.
// (For signing, we must push to atomic: to get a Docker identity to use in the signature.)
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--policy", policy, "copy", "--sign-by", "personal@example.com", topDirDest+"/dir1", "atomic:localhost:5000/myns/personal:dirstaging")
@@ -994,7 +977,7 @@ func (s *CopySuite) TestCopyDockerSigstore(c *check.C) {
c.Assert(err, check.IsNil)
// Get an image to work with. Also verifies that we can use Docker repositories with no sigstore configured.
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--registries.d", registriesDir, "copy", testFQIN, ourRegistry+"original/busybox")
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--registries.d", registriesDir, "copy", "docker://busybox", ourRegistry+"original/busybox")
// Pulling an unsigned image fails.
assertSkopeoFails(c, ".*Source image rejected: A signature was required, but no signature exists.*",
"--tls-verify=false", "--policy", policy, "--registries.d", registriesDir, "copy", ourRegistry+"original/busybox", dirDest)
@@ -1048,7 +1031,7 @@ func (s *CopySuite) TestCopyAtomicExtension(c *check.C) {
defer os.Remove(policy)
// Get an image to work with to an atomic: destination. Also verifies that we can use Docker repositories without X-Registry-Supports-Signatures
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--registries.d", registriesDir, "copy", testFQIN, "atomic:localhost:5000/myns/extension:unsigned")
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--registries.d", registriesDir, "copy", "docker://busybox", "atomic:localhost:5000/myns/extension:unsigned")
// Pulling an unsigned image using atomic: fails.
assertSkopeoFails(c, ".*Source image rejected: A signature was required, but no signature exists.*",
"--tls-verify=false", "--policy", policy,
@@ -1072,7 +1055,7 @@ func (s *CopySuite) TestCopyAtomicExtension(c *check.C) {
// Get another image (different so that they don't share signatures, and sign it using docker://)
assertSkopeoSucceeds(c, "", "--tls-verify=false", "--registries.d", registriesDir,
"copy", "--sign-by", "personal@example.com", testFQIN+":ppc64le", "docker://localhost:5000/myns/extension:extension")
"copy", "--sign-by", "personal@example.com", "docker://estesp/busybox:ppc64le", "docker://localhost:5000/myns/extension:extension")
c.Logf("%s", combinedOutputOfCommand(c, "oc", "get", "istag", "extension:extension", "-o", "json"))
// Pulling the image using atomic: succeeds.
assertSkopeoSucceeds(c, "", "--debug", "--tls-verify=false", "--policy", policy,
@@ -1084,23 +1067,6 @@ func (s *CopySuite) TestCopyAtomicExtension(c *check.C) {
assertDirImagesAreEqual(c, filepath.Join(topDir, "dirDA"), filepath.Join(topDir, "dirDD"))
}
// copyWithSignedIdentity creates a copy of an unsigned image, adding a signature for an unrelated identity
// This should be easier than using standalone-sign.
func copyWithSignedIdentity(c *check.C, src, dest, signedIdentity, signBy, registriesDir string) {
topDir, err := ioutil.TempDir("", "copyWithSignedIdentity")
c.Assert(err, check.IsNil)
defer os.RemoveAll(topDir)
signingDir := filepath.Join(topDir, "signing-temp")
assertSkopeoSucceeds(c, "", "copy", "--src-tls-verify=false", src, "dir:"+signingDir)
c.Logf("%s", combinedOutputOfCommand(c, "ls", "-laR", signingDir))
assertSkopeoSucceeds(c, "^$", "standalone-sign", "-o", filepath.Join(signingDir, "signature-1"),
filepath.Join(signingDir, "manifest.json"), signedIdentity, signBy)
c.Logf("%s", combinedOutputOfCommand(c, "ls", "-laR", signingDir))
assertSkopeoSucceeds(c, "", "--registries.d", registriesDir, "copy", "--dest-tls-verify=false", "dir:"+signingDir, dest)
}
// Both mirroring support in registries.conf, and mirrored remapIdentity support in policy.json
func (s *CopySuite) TestCopyVerifyingMirroredSignatures(c *check.C) {
const regPrefix = "docker://localhost:5006/myns/mirroring-"
@@ -1111,7 +1077,7 @@ func (s *CopySuite) TestCopyVerifyingMirroredSignatures(c *check.C) {
c.Skip(fmt.Sprintf("Signing not supported: %v", err))
}
topDir, err := ioutil.TempDir("", "mirrored-signatures")
topDir, err := ioutil.TempDir("", "mirrored-signatures") // FIXME: Will this be used?
c.Assert(err, check.IsNil)
defer os.RemoveAll(topDir)
registriesDir := filepath.Join(topDir, "registries.d") // An empty directory to disable sigstore use
@@ -1121,12 +1087,12 @@ func (s *CopySuite) TestCopyVerifyingMirroredSignatures(c *check.C) {
defer os.Remove(policy)
// We use X-R-S-S for this testing to avoid having to deal with the sigstores.
// A downside is that OpenShift records signatures per image, so the error messages below
// A downside is that OpenShift records signatures per image, so the error messsages below
// list all signatures for other tags used for the same image as well.
// So, make sure to never create a signature that could be considered valid in a different part of the test (i.e. don't reuse tags).
// Get an image to work with.
assertSkopeoSucceeds(c, "", "copy", "--dest-tls-verify=false", testFQIN, regPrefix+"primary:unsigned")
assertSkopeoSucceeds(c, "", "copy", "--dest-tls-verify=false", "docker://busybox", regPrefix+"primary:unsigned")
// Verify that unsigned images are rejected
assertSkopeoFails(c, ".*Source image rejected: A signature was required, but no signature exists.*",
"--policy", policy, "--registries.d", registriesDir, "--registries-conf", "fixtures/registries.conf", "copy", "--src-tls-verify=false", regPrefix+"primary:unsigned", dirDest)
@@ -1137,7 +1103,7 @@ func (s *CopySuite) TestCopyVerifyingMirroredSignatures(c *check.C) {
// Sign the image for the mirror
assertSkopeoSucceeds(c, "", "--registries.d", registriesDir, "copy", "--src-tls-verify=false", "--dest-tls-verify=false", "--sign-by", "personal@example.com", regPrefix+"primary:unsigned", regPrefix+"mirror:mirror-signed")
// Verify that a correctly signed image for the mirror is accessible using the mirror's reference
// Verify that a correctly signed image for the mirror is acessible using the mirror's reference
assertSkopeoSucceeds(c, "", "--policy", policy, "--registries.d", registriesDir, "--registries-conf", "fixtures/registries.conf", "copy", "--src-tls-verify=false", regPrefix+"mirror:mirror-signed", dirDest)
// … but verify that while it is accessible using the primary location redirecting to the mirror, …
assertSkopeoSucceeds(c, "" /* no --policy */, "--registries-conf", "fixtures/registries.conf", "copy", "--src-tls-verify=false", regPrefix+"primary:mirror-signed", dirDest)
@@ -1145,10 +1111,14 @@ func (s *CopySuite) TestCopyVerifyingMirroredSignatures(c *check.C) {
assertSkopeoFails(c, ".*Source image rejected: None of the signatures were accepted, reasons: Signature for identity localhost:5006/myns/mirroring-primary:direct is not accepted; Signature for identity localhost:5006/myns/mirroring-mirror:mirror-signed is not accepted.*",
"--policy", policy, "--registries.d", registriesDir, "--registries-conf", "fixtures/registries.conf", "copy", "--src-tls-verify=false", regPrefix+"primary:mirror-signed", dirDest)
// Create a signature for mirroring-primary:primary-signed without pushing there.
copyWithSignedIdentity(c, regPrefix+"primary:unsigned", regPrefix+"mirror:primary-signed",
"localhost:5006/myns/mirroring-primary:primary-signed", "personal@example.com",
registriesDir)
// Create a signature for mirroring-primary:primary-signed without pushing there. This should be easier than using standalone-sign.
signingDir := filepath.Join(topDir, "signing-temp")
assertSkopeoSucceeds(c, "", "copy", "--src-tls-verify=false", regPrefix+"primary:unsigned", "dir:"+signingDir)
c.Logf("%s", combinedOutputOfCommand(c, "ls", "-laR", signingDir))
assertSkopeoSucceeds(c, "^$", "standalone-sign", "-o", filepath.Join(signingDir, "signature-1"),
filepath.Join(signingDir, "manifest.json"), "localhost:5006/myns/mirroring-primary:primary-signed", "personal@example.com")
c.Logf("%s", combinedOutputOfCommand(c, "ls", "-laR", signingDir))
assertSkopeoSucceeds(c, "", "--registries.d", registriesDir, "copy", "--dest-tls-verify=false", "dir:"+signingDir, regPrefix+"mirror:primary-signed")
// Verify that a correctly signed image for the primary is accessible using the primary's reference
assertSkopeoSucceeds(c, "", "--policy", policy, "--registries.d", registriesDir, "--registries-conf", "fixtures/registries.conf", "copy", "--src-tls-verify=false", regPrefix+"primary:primary-signed", dirDest)
// … but verify that while it is accessible using the mirror location
@@ -1156,24 +1126,10 @@ func (s *CopySuite) TestCopyVerifyingMirroredSignatures(c *check.C) {
// … verify it is NOT accessible when requiring a signature.
assertSkopeoFails(c, ".*Source image rejected: None of the signatures were accepted, reasons: Signature for identity localhost:5006/myns/mirroring-primary:direct is not accepted; Signature for identity localhost:5006/myns/mirroring-mirror:mirror-signed is not accepted; Signature for identity localhost:5006/myns/mirroring-primary:primary-signed is not accepted.*",
"--policy", policy, "--registries.d", registriesDir, "--registries-conf", "fixtures/registries.conf", "copy", "--src-tls-verify=false", regPrefix+"mirror:primary-signed", dirDest)
assertSkopeoSucceeds(c, "", "--registries.d", registriesDir, "--registries-conf", "fixtures/registries.conf", "copy", "--src-tls-verify=false", "--dest-tls-verify=false", regPrefix+"primary:unsigned", regPrefix+"remap:remapped")
// Verify that while a remapIdentity image is accessible using the remapped (mirror) location
assertSkopeoSucceeds(c, "" /* no --policy */, "--registries.d", registriesDir, "--registries-conf", "fixtures/registries.conf", "copy", "--src-tls-verify=false", regPrefix+"remap:remapped", dirDest)
// … it is NOT accessible when requiring a signature …
assertSkopeoFails(c, ".*Source image rejected: None of the signatures were accepted, reasons: Signature for identity localhost:5006/myns/mirroring-primary:direct is not accepted; Signature for identity localhost:5006/myns/mirroring-mirror:mirror-signed is not accepted; Signature for identity localhost:5006/myns/mirroring-primary:primary-signed is not accepted.*", "--policy", policy, "--registries.d", registriesDir, "--registries-conf", "fixtures/registries.conf", "copy", "--src-tls-verify=false", regPrefix+"remap:remapped", dirDest)
// … until signed.
copyWithSignedIdentity(c, regPrefix+"remap:remapped", regPrefix+"remap:remapped",
"localhost:5006/myns/mirroring-primary:remapped", "personal@example.com",
registriesDir)
assertSkopeoSucceeds(c, "", "--policy", policy, "--registries.d", registriesDir, "--registries-conf", "fixtures/registries.conf", "copy", "--src-tls-verify=false", regPrefix+"remap:remapped", dirDest)
// To be extra clear about the semantics, verify that the signedPrefix (primary) location never exists
// and only the remapped prefix (mirror) is accessed.
assertSkopeoFails(c, ".*initializing source docker://localhost:5006/myns/mirroring-primary:remapped:.*manifest unknown: manifest unknown.*", "--policy", policy, "--registries.d", registriesDir, "--registries-conf", "fixtures/registries.conf", "copy", "--src-tls-verify=false", regPrefix+"primary:remapped", dirDest)
}
func (s *SkopeoSuite) TestCopySrcWithAuth(c *check.C) {
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "--dest-creds=testuser:testpassword", testFQIN, fmt.Sprintf("docker://%s/busybox:latest", s.regV2WithAuth.url))
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "--dest-creds=testuser:testpassword", "docker://busybox", fmt.Sprintf("docker://%s/busybox:latest", s.regV2WithAuth.url))
dir1, err := ioutil.TempDir("", "copy-1")
c.Assert(err, check.IsNil)
defer os.RemoveAll(dir1)
@@ -1181,15 +1137,15 @@ func (s *SkopeoSuite) TestCopySrcWithAuth(c *check.C) {
}
func (s *SkopeoSuite) TestCopyDestWithAuth(c *check.C) {
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "--dest-creds=testuser:testpassword", testFQIN, fmt.Sprintf("docker://%s/busybox:latest", s.regV2WithAuth.url))
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "--dest-creds=testuser:testpassword", "docker://busybox", fmt.Sprintf("docker://%s/busybox:latest", s.regV2WithAuth.url))
}
func (s *SkopeoSuite) TestCopySrcAndDestWithAuth(c *check.C) {
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "--dest-creds=testuser:testpassword", testFQIN, fmt.Sprintf("docker://%s/busybox:latest", s.regV2WithAuth.url))
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "--dest-creds=testuser:testpassword", "docker://busybox", fmt.Sprintf("docker://%s/busybox:latest", s.regV2WithAuth.url))
assertSkopeoSucceeds(c, "", "--tls-verify=false", "copy", "--src-creds=testuser:testpassword", "--dest-creds=testuser:testpassword", fmt.Sprintf("docker://%s/busybox:latest", s.regV2WithAuth.url), fmt.Sprintf("docker://%s/test:auth", s.regV2WithAuth.url))
}
func (s *CopySuite) TestCopyNoPanicOnHTTPResponseWithoutTLSVerifyFalse(c *check.C) {
func (s *CopySuite) TestCopyNoPanicOnHTTPResponseWOTLSVerifyFalse(c *check.C) {
const ourRegistry = "docker://" + v2DockerRegistryURL + "/"
// dir:test isn't created beforehand just because we already know this could
@@ -1215,7 +1171,7 @@ func (s *CopySuite) TestCopyManifestConversion(c *check.C) {
// oci to v2s1 and vice-versa not supported yet
// get v2s2 manifest type
assertSkopeoSucceeds(c, "", "copy", testFQIN, "dir:"+srcDir)
assertSkopeoSucceeds(c, "", "copy", "docker://busybox", "dir:"+srcDir)
verifyManifestMIMEType(c, srcDir, manifest.DockerV2Schema2MediaType)
// convert from v2s2 to oci
assertSkopeoSucceeds(c, "", "copy", "--format=oci", "dir:"+srcDir, "dir:"+destDir1)
@@ -1245,7 +1201,7 @@ func (s *CopySuite) testCopySchemaConversionRegistries(c *check.C, schema1Regist
// Ensure we are working with a schema2 image.
// dir: accepts any manifest format, i.e. this makes …/input2 a schema2 source which cannot be asked to produce schema1 like ordinary docker: registries can.
assertSkopeoSucceeds(c, "", "copy", testFQIN, "dir:"+input2Dir)
assertSkopeoSucceeds(c, "", "copy", "docker://busybox", "dir:"+input2Dir)
verifyManifestMIMEType(c, input2Dir, manifest.DockerV2Schema2MediaType)
// 2→2 (the "f2t2" in tag means "from 2 to 2")
assertSkopeoSucceeds(c, "", "copy", "--dest-tls-verify=false", "dir:"+input2Dir, schema2Registry+":f2t2")
@@ -1265,6 +1221,14 @@ func (s *CopySuite) testCopySchemaConversionRegistries(c *check.C, schema1Regist
verifyManifestMIMEType(c, destDir, manifest.DockerV2Schema1SignedMediaType)
}
// Verify manifest in a dir: image at dir is expectedMIMEType.
func verifyManifestMIMEType(c *check.C, dir string, expectedMIMEType string) {
manifestBlob, err := ioutil.ReadFile(filepath.Join(dir, "manifest.json"))
c.Assert(err, check.IsNil)
mimeType := manifest.GuessMIMEType(manifestBlob)
c.Assert(mimeType, check.Equals, expectedMIMEType)
}
const regConfFixture = "./fixtures/registries.conf"
func (s *SkopeoSuite) TestSuccessCopySrcWithMirror(c *check.C) {

View File

@@ -34,27 +34,10 @@
"keyPath": "@keydir@/personal-pubkey.gpg"
}
],
"localhost:5006/myns/mirroring-remap": [
{
"type": "signedBy",
"keyType": "GPGKeys",
"keyPath": "@keydir@/personal-pubkey.gpg",
"signedIdentity": {
"type": "remapIdentity",
"prefix": "localhost:5006/myns/mirroring-remap",
"signedPrefix": "localhost:5006/myns/mirroring-primary"
}
}
],
"docker.io/openshift": [
{
"type": "insecureAcceptAnything"
}
],
"quay.io/openshift": [
{
"type": "insecureAcceptAnything"
}
]
},
"dir": {

View File

@@ -2,7 +2,6 @@ package main
import (
"bufio"
"context"
"encoding/base64"
"fmt"
"io/ioutil"
@@ -10,7 +9,6 @@ import (
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/docker/docker/pkg/homedir"
"github.com/go-check/check"
@@ -64,7 +62,6 @@ func (cluster *openshiftCluster) startMaster(c *check.C) {
cmd := cluster.clusterCmd(nil, "openshift", "start", "master")
cluster.processes = append(cluster.processes, cmd)
stdout, err := cmd.StdoutPipe()
c.Assert(err, check.IsNil)
// Send both to the same pipe. This might cause the two streams to be mixed up,
// but logging actually goes only to stderr - this primarily ensure we log any
// unexpected output to stdout.
@@ -111,8 +108,6 @@ func (cluster *openshiftCluster) startMaster(c *check.C) {
gotPortCheck := false
gotLogCheck := false
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
for !gotPortCheck || !gotLogCheck {
c.Logf("Waiting for master")
select {
@@ -125,8 +120,6 @@ func (cluster *openshiftCluster) startMaster(c *check.C) {
c.Fatal("log check done, success message not found")
}
gotLogCheck = true
case <-ctx.Done():
c.Fatalf("Timed out waiting for master: %v", ctx.Err())
}
}
c.Logf("OK, master started!")
@@ -172,14 +165,8 @@ func (cluster *openshiftCluster) startRegistryProcess(c *check.C, port int, conf
terminatePortCheck <- true
}()
c.Logf("Waiting for registry to start")
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
select {
case <-portOpen:
c.Logf("OK, Registry port open")
case <-ctx.Done():
c.Fatalf("Timed out waiting for registry to start: %v", ctx.Err())
}
<-portOpen
c.Logf("OK, Registry port open")
return cmd
}

View File

@@ -1,4 +1,3 @@
//go:build openshift_shell
// +build openshift_shell
package main
@@ -20,8 +19,8 @@ to start a container, then within the container:
SKOPEO_CONTAINER_TESTS=1 PS1='nested> ' go test -tags openshift_shell -timeout=24h ./integration -v -check.v -check.vv -check.f='CopySuite.TestRunShell'
An example of what can be done within the container:
cd ..; make bin/skopeo PREFIX=/usr install
./skopeo --tls-verify=false copy --sign-by=personal@example.com docker://quay.io/libpod/busybox:latest atomic:localhost:5000/myns/personal:personal
cd ..; make binary-local install
./skopeo --tls-verify=false copy --sign-by=personal@example.com docker://busybox:latest atomic:localhost:5000/myns/personal:personal
oc get istag personal:personal -o json
curl -L -v 'http://localhost:5000/v2/'
cat ~/.docker/config.json

View File

@@ -11,26 +11,8 @@ import (
"strings"
"github.com/containers/image/v5/docker"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/types"
"github.com/go-check/check"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
)
const (
// A repository with a path with multiple components in it which
// contains multiple tags, preferably with some tags pointing to
// manifest lists, and with some tags that don't.
pullableRepo = "quay.io/libpod/testimage"
// A tagged image in the repository that we can inspect and copy.
pullableTaggedImage = "k8s.gcr.io/coredns/coredns:v1.6.6"
// A tagged manifest list in the repository that we can inspect and copy.
pullableTaggedManifestList = "k8s.gcr.io/coredns/coredns:v1.8.0"
// A repository containing multiple tags, some of which are for
// manifest lists, and which includes a "latest" tag. We specify the
// name here without a tag.
pullableRepoWithLatestTag = "k8s.gcr.io/pause"
)
func init() {
@@ -112,8 +94,8 @@ func (s *SyncSuite) TestDocker2DirTagged(c *check.C) {
c.Assert(err, check.IsNil)
defer os.RemoveAll(tmpDir)
// FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection.
image := pullableTaggedImage
// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
image := "busybox:latest"
imageRef, err := docker.ParseReference(fmt.Sprintf("//%s", image))
c.Assert(err, check.IsNil)
imagePath := imageRef.DockerReference().String()
@@ -135,37 +117,9 @@ func (s *SyncSuite) TestDocker2DirTagged(c *check.C) {
c.Assert(out, check.Equals, "")
}
func (s *SyncSuite) TestDocker2DirTaggedAll(c *check.C) {
tmpDir, err := ioutil.TempDir("", "skopeo-sync-test")
c.Assert(err, check.IsNil)
defer os.RemoveAll(tmpDir)
// FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection.
image := pullableTaggedManifestList
imageRef, err := docker.ParseReference(fmt.Sprintf("//%s", image))
c.Assert(err, check.IsNil)
imagePath := imageRef.DockerReference().String()
dir1 := path.Join(tmpDir, "dir1")
dir2 := path.Join(tmpDir, "dir2")
// sync docker => dir
assertSkopeoSucceeds(c, "", "sync", "--all", "--scoped", "--src", "docker", "--dest", "dir", image, dir1)
_, err = os.Stat(path.Join(dir1, imagePath, "manifest.json"))
c.Assert(err, check.IsNil)
// copy docker => dir
assertSkopeoSucceeds(c, "", "copy", "--all", "docker://"+image, "dir:"+dir2)
_, err = os.Stat(path.Join(dir2, "manifest.json"))
c.Assert(err, check.IsNil)
out := combinedOutputOfCommand(c, "diff", "-urN", path.Join(dir1, imagePath), dir2)
c.Assert(out, check.Equals, "")
}
func (s *SyncSuite) TestScoped(c *check.C) {
// FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection.
image := pullableTaggedImage
// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
image := "busybox:latest"
imageRef, err := docker.ParseReference(fmt.Sprintf("//%s", image))
c.Assert(err, check.IsNil)
imagePath := imageRef.DockerReference().String()
@@ -173,7 +127,7 @@ func (s *SyncSuite) TestScoped(c *check.C) {
dir1, err := ioutil.TempDir("", "skopeo-sync-test")
c.Assert(err, check.IsNil)
assertSkopeoSucceeds(c, "", "sync", "--src", "docker", "--dest", "dir", image, dir1)
_, err = os.Stat(path.Join(dir1, path.Base(imagePath), "manifest.json"))
_, err = os.Stat(path.Join(dir1, image, "manifest.json"))
c.Assert(err, check.IsNil)
assertSkopeoSucceeds(c, "", "sync", "--scoped", "--src", "docker", "--dest", "dir", image, dir1)
@@ -184,30 +138,30 @@ func (s *SyncSuite) TestScoped(c *check.C) {
}
func (s *SyncSuite) TestDirIsNotOverwritten(c *check.C) {
// FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection.
image := pullableRepoWithLatestTag
// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
image := "busybox:latest"
imageRef, err := docker.ParseReference(fmt.Sprintf("//%s", image))
c.Assert(err, check.IsNil)
imagePath := imageRef.DockerReference().String()
// make a copy of the image in the local registry
assertSkopeoSucceeds(c, "", "copy", "--dest-tls-verify=false", "docker://"+image, "docker://"+path.Join(v2DockerRegistryURL, reference.Path(imageRef.DockerReference())))
assertSkopeoSucceeds(c, "", "copy", "--dest-tls-verify=false", "docker://"+image, "docker://"+path.Join(v2DockerRegistryURL, image))
//sync upstream image to dir, not scoped
dir1, err := ioutil.TempDir("", "skopeo-sync-test")
c.Assert(err, check.IsNil)
assertSkopeoSucceeds(c, "", "sync", "--src", "docker", "--dest", "dir", image, dir1)
_, err = os.Stat(path.Join(dir1, path.Base(imagePath), "manifest.json"))
_, err = os.Stat(path.Join(dir1, image, "manifest.json"))
c.Assert(err, check.IsNil)
//sync local registry image to dir, not scoped
assertSkopeoFails(c, ".*Refusing to overwrite destination directory.*", "sync", "--src-tls-verify=false", "--src", "docker", "--dest", "dir", path.Join(v2DockerRegistryURL, reference.Path(imageRef.DockerReference())), dir1)
assertSkopeoFails(c, ".*Refusing to overwrite destination directory.*", "sync", "--src-tls-verify=false", "--src", "docker", "--dest", "dir", path.Join(v2DockerRegistryURL, image), dir1)
//sync local registry image to dir, scoped
imageRef, err = docker.ParseReference(fmt.Sprintf("//%s", path.Join(v2DockerRegistryURL, reference.Path(imageRef.DockerReference()))))
imageRef, err = docker.ParseReference(fmt.Sprintf("//%s", path.Join(v2DockerRegistryURL, image)))
c.Assert(err, check.IsNil)
imagePath = imageRef.DockerReference().String()
assertSkopeoSucceeds(c, "", "sync", "--scoped", "--src-tls-verify=false", "--src", "docker", "--dest", "dir", path.Join(v2DockerRegistryURL, reference.Path(imageRef.DockerReference())), dir1)
assertSkopeoSucceeds(c, "", "sync", "--scoped", "--src-tls-verify=false", "--src", "docker", "--dest", "dir", path.Join(v2DockerRegistryURL, image), dir1)
_, err = os.Stat(path.Join(dir1, imagePath, "manifest.json"))
c.Assert(err, check.IsNil)
os.RemoveAll(dir1)
@@ -219,8 +173,8 @@ func (s *SyncSuite) TestDocker2DirUntagged(c *check.C) {
c.Assert(err, check.IsNil)
defer os.RemoveAll(tmpDir)
// FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection.
image := pullableRepo
// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
image := "alpine"
imageRef, err := docker.ParseReference(fmt.Sprintf("//%s", image))
c.Assert(err, check.IsNil)
imagePath := imageRef.DockerReference().String()
@@ -244,7 +198,7 @@ func (s *SyncSuite) TestYamlUntagged(c *check.C) {
defer os.RemoveAll(tmpDir)
dir1 := path.Join(tmpDir, "dir1")
image := pullableRepo
image := "alpine"
imageRef, err := docker.ParseReference(fmt.Sprintf("//%s", image))
c.Assert(err, check.IsNil)
imagePath := imageRef.DockerReference().Name()
@@ -255,22 +209,23 @@ func (s *SyncSuite) TestYamlUntagged(c *check.C) {
c.Check(len(tags), check.Not(check.Equals), 0)
yamlConfig := fmt.Sprintf(`
%s:
docker.io:
images:
%s: []
`, reference.Domain(imageRef.DockerReference()), reference.Path(imageRef.DockerReference()))
%s:
`, image)
// sync to the local registry
//sync to the local reg
yamlFile := path.Join(tmpDir, "registries.yaml")
ioutil.WriteFile(yamlFile, []byte(yamlConfig), 0644)
assertSkopeoSucceeds(c, "", "sync", "--scoped", "--src", "yaml", "--dest", "docker", "--dest-tls-verify=false", yamlFile, v2DockerRegistryURL)
// sync back from local registry to a folder
// sync back from local reg to a folder
os.Remove(yamlFile)
yamlConfig = fmt.Sprintf(`
%s:
tls-verify: false
images:
%s: []
%s:
`, v2DockerRegistryURL, imagePath)
ioutil.WriteFile(yamlFile, []byte(yamlConfig), 0644)
@@ -279,9 +234,7 @@ func (s *SyncSuite) TestYamlUntagged(c *check.C) {
sysCtx = types.SystemContext{
DockerInsecureSkipTLSVerify: types.NewOptionalBool(true),
}
localImageRef, err := docker.ParseReference(fmt.Sprintf("//%s/%s", v2DockerRegistryURL, imagePath))
c.Assert(err, check.IsNil)
localTags, err := docker.GetRepositoryTags(context.Background(), &sysCtx, localImageRef)
localTags, err := docker.GetRepositoryTags(context.Background(), &sysCtx, imageRef)
c.Assert(err, check.IsNil)
c.Check(len(localTags), check.Not(check.Equals), 0)
c.Assert(len(localTags), check.Equals, len(tags))
@@ -309,9 +262,9 @@ func (s *SyncSuite) TestYamlRegex2Dir(c *check.C) {
dir1 := path.Join(tmpDir, "dir1")
yamlConfig := `
k8s.gcr.io:
docker.io:
images-by-tag-regex:
pause: ^[12]\.0$ # regex string test
nginx: ^1\.13\.[12]-alpine-perl$ # regex string test
`
// the ↑ regex strings always matches only 2 images
var nTags = 2
@@ -336,37 +289,6 @@ k8s.gcr.io:
c.Assert(nManifests, check.Equals, nTags)
}
func (s *SyncSuite) TestYamlDigest2Dir(c *check.C) {
tmpDir, err := ioutil.TempDir("", "skopeo-sync-test")
c.Assert(err, check.IsNil)
defer os.RemoveAll(tmpDir)
dir1 := path.Join(tmpDir, "dir1")
yamlConfig := `
k8s.gcr.io:
images:
pause:
- sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610
`
yamlFile := path.Join(tmpDir, "registries.yaml")
ioutil.WriteFile(yamlFile, []byte(yamlConfig), 0644)
assertSkopeoSucceeds(c, "", "sync", "--scoped", "--src", "yaml", "--dest", "dir", yamlFile, dir1)
nManifests := 0
err = filepath.Walk(dir1, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && info.Name() == "manifest.json" {
nManifests++
return filepath.SkipDir
}
return nil
})
c.Assert(err, check.IsNil)
c.Assert(nManifests, check.Equals, 1)
}
func (s *SyncSuite) TestYaml2Dir(c *check.C) {
tmpDir, err := ioutil.TempDir("", "skopeo-sync-test")
c.Assert(err, check.IsNil)
@@ -374,21 +296,21 @@ func (s *SyncSuite) TestYaml2Dir(c *check.C) {
dir1 := path.Join(tmpDir, "dir1")
yamlConfig := `
k8s.gcr.io:
docker.io:
images:
coredns/coredns:
- v1.8.0
- v1.7.1
k8s-dns-kube-dns:
- 1.14.12
- 1.14.13
pause:
busybox:
- latest
- musl
alpine:
- edge
- 3.8
opensuse/leap:
- latest
quay.io:
images:
quay/busybox:
- latest`
quay/busybox:
- latest`
// get the number of tags
re := regexp.MustCompile(`^ +- +[^:/ ]+`)
@@ -425,10 +347,10 @@ func (s *SyncSuite) TestYamlTLSVerify(c *check.C) {
c.Assert(err, check.IsNil)
defer os.RemoveAll(tmpDir)
dir1 := path.Join(tmpDir, "dir1")
image := pullableRepoWithLatestTag
image := "busybox"
tag := "latest"
// FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection.
// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
// copy docker => docker
assertSkopeoSucceeds(c, "", "copy", "--dest-tls-verify=false", "docker://"+image+":"+tag, localRegURL+image+":"+tag)
@@ -474,26 +396,6 @@ func (s *SyncSuite) TestYamlTLSVerify(c *check.C) {
}
func (s *SyncSuite) TestSyncManifestOutput(c *check.C) {
tmpDir, err := ioutil.TempDir("", "sync-manifest-output")
c.Assert(err, check.IsNil)
defer os.RemoveAll(tmpDir)
destDir1 := filepath.Join(tmpDir, "dest1")
destDir2 := filepath.Join(tmpDir, "dest2")
destDir3 := filepath.Join(tmpDir, "dest3")
//Split image:tag path from image URI for manifest comparison
imageDir := pullableTaggedImage[strings.LastIndex(pullableTaggedImage, "/")+1:]
assertSkopeoSucceeds(c, "", "sync", "--format=oci", "--all", "--src", "docker", "--dest", "dir", pullableTaggedImage, destDir1)
verifyManifestMIMEType(c, filepath.Join(destDir1, imageDir), imgspecv1.MediaTypeImageManifest)
assertSkopeoSucceeds(c, "", "sync", "--format=v2s2", "--all", "--src", "docker", "--dest", "dir", pullableTaggedImage, destDir2)
verifyManifestMIMEType(c, filepath.Join(destDir2, imageDir), manifest.DockerV2Schema2MediaType)
assertSkopeoSucceeds(c, "", "sync", "--format=v2s1", "--all", "--src", "docker", "--dest", "dir", pullableTaggedImage, destDir3)
verifyManifestMIMEType(c, filepath.Join(destDir3, imageDir), manifest.DockerV2Schema1SignedMediaType)
}
func (s *SyncSuite) TestDocker2DockerTagged(c *check.C) {
const localRegURL = "docker://" + v2DockerRegistryURL + "/"
@@ -501,8 +403,8 @@ func (s *SyncSuite) TestDocker2DockerTagged(c *check.C) {
c.Assert(err, check.IsNil)
defer os.RemoveAll(tmpDir)
// FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection.
image := pullableTaggedImage
// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
image := "busybox:latest"
imageRef, err := docker.ParseReference(fmt.Sprintf("//%s", image))
c.Assert(err, check.IsNil)
imagePath := imageRef.DockerReference().String()
@@ -534,8 +436,8 @@ func (s *SyncSuite) TestDir2DockerTagged(c *check.C) {
c.Assert(err, check.IsNil)
defer os.RemoveAll(tmpDir)
// FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection.
image := pullableRepoWithLatestTag
// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
image := "busybox:latest"
dir1 := path.Join(tmpDir, "dir1")
err = os.Mkdir(dir1, 0755)
@@ -544,10 +446,6 @@ func (s *SyncSuite) TestDir2DockerTagged(c *check.C) {
err = os.Mkdir(dir2, 0755)
c.Assert(err, check.IsNil)
// create leading dirs
err = os.MkdirAll(path.Dir(path.Join(dir1, image)), 0755)
c.Assert(err, check.IsNil)
// copy docker => dir
assertSkopeoSucceeds(c, "", "copy", "docker://"+image, "dir:"+path.Join(dir1, image))
_, err = os.Stat(path.Join(dir1, image, "manifest.json"))
@@ -556,13 +454,9 @@ func (s *SyncSuite) TestDir2DockerTagged(c *check.C) {
// sync dir => docker
assertSkopeoSucceeds(c, "", "sync", "--scoped", "--dest-tls-verify=false", "--src", "dir", "--dest", "docker", dir1, v2DockerRegistryURL)
// create leading dirs
err = os.MkdirAll(path.Dir(path.Join(dir2, image)), 0755)
c.Assert(err, check.IsNil)
// copy docker => dir
assertSkopeoSucceeds(c, "", "copy", "--src-tls-verify=false", localRegURL+image, "dir:"+path.Join(dir2, image))
_, err = os.Stat(path.Join(dir2, image, "manifest.json"))
_, err = os.Stat(path.Join(path.Join(dir2, image), "manifest.json"))
c.Assert(err, check.IsNil)
out := combinedOutputOfCommand(c, "diff", "-urN", dir1, dir2)
@@ -625,7 +519,7 @@ func (s *SyncSuite) TestFailsWithDockerSourceUnauthorized(c *check.C) {
}
func (s *SyncSuite) TestFailsWithDockerSourceNotExisting(c *check.C) {
repo := path.Join(v2DockerRegistryURL, "imagedoesnotexist")
repo := path.Join(v2DockerRegistryURL, "imagedoesdotexist")
tmpDir, err := ioutil.TempDir("", "skopeo-sync-test")
c.Assert(err, check.IsNil)
defer os.RemoveAll(tmpDir)
@@ -635,7 +529,7 @@ func (s *SyncSuite) TestFailsWithDockerSourceNotExisting(c *check.C) {
"sync", "--scoped", "--src-tls-verify=false", "--src", "docker", "--dest", "dir", repo, tmpDir)
//tagged
assertSkopeoFails(c, ".*reading manifest.*",
assertSkopeoFails(c, ".*Error reading manifest.*",
"sync", "--scoped", "--src-tls-verify=false", "--src", "docker", "--dest", "dir", repo+":thetag", tmpDir)
}

View File

@@ -10,17 +10,12 @@ import (
"strings"
"time"
"github.com/containers/image/v5/manifest"
"github.com/go-check/check"
)
const skopeoBinary = "skopeo"
const decompressDirsBinary = "./decompress-dirs.sh"
const testFQIN = "docker://quay.io/libpod/busybox" // tag left off on purpose, some tests need to add a special one
const testFQIN64 = "docker://quay.io/libpod/busybox:amd64"
const testFQINMultiLayer = "docker://quay.io/libpod/alpine_nginx:master" // multi-layer
// consumeAndLogOutputStream takes (f, err) from an exec.*Pipe(), and causes all output to it to be logged to c.
func consumeAndLogOutputStream(c *check.C, id string, f io.ReadCloser, err error) {
c.Assert(err, check.IsNil)
@@ -205,11 +200,3 @@ func runDecompressDirs(c *check.C, regexp string, args ...string) {
c.Assert(string(out), check.Matches, "(?s)"+regexp) // (?s) : '.' will also match newlines
}
}
// Verify manifest in a dir: image at dir is expectedMIMEType.
func verifyManifestMIMEType(c *check.C, dir string, expectedMIMEType string) {
manifestBlob, err := ioutil.ReadFile(filepath.Join(dir, "manifest.json"))
c.Assert(err, check.IsNil)
mimeType := manifest.GuessMIMEType(manifestBlob)
c.Assert(mimeType, check.Equals, expectedMIMEType)
}

View File

@@ -1,85 +0,0 @@
let
pkgs = (import ./nixpkgs.nix {
crossSystem = {
config = "aarch64-unknown-linux-gnu";
};
config = {
packageOverrides = pkg: {
gpgme = (static pkg.gpgme);
libassuan = (static pkg.libassuan);
libgpgerror = (static pkg.libgpgerror);
libseccomp = (static pkg.libseccomp);
glib = (static pkg.glib).overrideAttrs (x: {
outputs = [ "bin" "out" "dev" ];
mesonFlags = [
"-Ddefault_library=static"
"-Ddevbindir=${placeholder ''dev''}/bin"
"-Dgtk_doc=false"
"-Dnls=disabled"
];
postInstall = ''
moveToOutput "share/glib-2.0" "$dev"
substituteInPlace "$dev/bin/gdbus-codegen" --replace "$out" "$dev"
sed -i "$dev/bin/glib-gettextize" -e "s|^gettext_dir=.*|gettext_dir=$dev/share/glib-2.0/gettext|"
sed '1i#line 1 "${x.pname}-${x.version}/include/glib-2.0/gobject/gobjectnotifyqueue.c"' \
-i "$dev"/include/glib-2.0/gobject/gobjectnotifyqueue.c
'';
});
pcsclite = (static pkg.pcsclite).overrideAttrs (x: {
configureFlags = [
"--enable-confdir=/etc"
"--enable-usbdropdir=/var/lib/pcsc/drivers"
"--disable-libsystemd"
"--disable-libudev"
"--disable-libusb"
];
buildInputs = [ pkgs.python3 pkgs.dbus ];
});
systemd = (static pkg.systemd).overrideAttrs (x: {
outputs = [ "out" "dev" ];
mesonFlags = x.mesonFlags ++ [
"-Dglib=false"
"-Dstatic-libsystemd=true"
];
});
};
};
});
static = pkg: pkg.overrideAttrs (x: {
doCheck = false;
configureFlags = (x.configureFlags or [ ]) ++ [
"--without-shared"
"--disable-shared"
];
dontDisableStatic = true;
enableSharedExecutables = false;
enableStatic = true;
});
self = with pkgs; buildGoModule rec {
name = "skopeo";
src = ./..;
vendorSha256 = null;
doCheck = false;
enableParallelBuilding = true;
outputs = [ "out" ];
nativeBuildInputs = [ bash gitMinimal go-md2man pkg-config which ];
buildInputs = [ glibc glibc.static glib gpgme libassuan libgpgerror libseccomp ];
prePatch = ''
export CFLAGS='-static -pthread'
export LDFLAGS='-s -w -static-libgcc -static'
export EXTRA_LDFLAGS='-s -w -linkmode external -extldflags "-static -lm"'
export BUILDTAGS='static netgo osusergo exclude_graphdriver_btrfs exclude_graphdriver_devicemapper'
export CGO_ENABLED=1
'';
buildPhase = ''
patchShebangs .
make bin/skopeo
'';
installPhase = ''
install -Dm755 bin/skopeo $out/bin/skopeo
'';
};
in
self

View File

@@ -1,83 +0,0 @@
{ system ? builtins.currentSystem }:
let
pkgs = (import ./nixpkgs.nix {
config = {
packageOverrides = pkg: {
gpgme = (static pkg.gpgme);
libassuan = (static pkg.libassuan);
libgpgerror = (static pkg.libgpgerror);
libseccomp = (static pkg.libseccomp);
glib = (static pkg.glib).overrideAttrs (x: {
outputs = [ "bin" "out" "dev" ];
mesonFlags = [
"-Ddefault_library=static"
"-Ddevbindir=${placeholder ''dev''}/bin"
"-Dgtk_doc=false"
"-Dnls=disabled"
];
postInstall = ''
moveToOutput "share/glib-2.0" "$dev"
substituteInPlace "$dev/bin/gdbus-codegen" --replace "$out" "$dev"
sed -i "$dev/bin/glib-gettextize" -e "s|^gettext_dir=.*|gettext_dir=$dev/share/glib-2.0/gettext|"
sed '1i#line 1 "${x.pname}-${x.version}/include/glib-2.0/gobject/gobjectnotifyqueue.c"' \
-i "$dev"/include/glib-2.0/gobject/gobjectnotifyqueue.c
'';
});
pcsclite = (static pkg.pcsclite).overrideAttrs (x: {
configureFlags = [
"--enable-confdir=/etc"
"--enable-usbdropdir=/var/lib/pcsc/drivers"
"--disable-libsystemd"
"--disable-libudev"
"--disable-libusb"
];
buildInputs = [ pkgs.python3 pkgs.dbus ];
});
systemd = (static pkg.systemd).overrideAttrs (x: {
outputs = [ "out" "dev" ];
mesonFlags = x.mesonFlags ++ [
"-Dglib=false"
"-Dstatic-libsystemd=true"
];
});
};
};
});
static = pkg: pkg.overrideAttrs (x: {
doCheck = false;
configureFlags = (x.configureFlags or [ ]) ++ [
"--without-shared"
"--disable-shared"
];
dontDisableStatic = true;
enableSharedExecutables = false;
enableStatic = true;
});
self = with pkgs; buildGoModule rec {
name = "skopeo";
src = ./..;
vendorSha256 = null;
doCheck = false;
enableParallelBuilding = true;
outputs = [ "out" ];
nativeBuildInputs = [ bash gitMinimal go-md2man pkg-config which ];
buildInputs = [ glibc glibc.static glib gpgme libassuan libgpgerror libseccomp ];
prePatch = ''
export CFLAGS='-static -pthread'
export LDFLAGS='-s -w -static-libgcc -static'
export EXTRA_LDFLAGS='-s -w -linkmode external -extldflags "-static -lm"'
export BUILDTAGS='static netgo osusergo exclude_graphdriver_btrfs exclude_graphdriver_devicemapper'
export CGO_ENABLED=1
'';
buildPhase = ''
patchShebangs .
make bin/skopeo
'';
installPhase = ''
install -Dm755 bin/skopeo $out/bin/skopeo
'';
};
in
self

View File

@@ -1,10 +0,0 @@
{
"url": "https://github.com/nixos/nixpkgs",
"rev": "2a96414d7e350160a33ed0978449c9ff5b5a6eb3",
"date": "2021-07-13T18:21:47+02:00",
"path": "/nix/store/2ai9q8ac6vxb2rrngdz82y8jxnk15cvm-nixpkgs",
"sha256": "1dzrfqdjq3yq5jjskiqflzy58l2xx6059gay9p1k07zrlm1wigy5",
"fetchSubmodules": false,
"deepClone": false,
"leaveDotGit": false
}

View File

@@ -1,9 +0,0 @@
let
json = builtins.fromJSON (builtins.readFile ./nixpkgs.json);
nixpkgs = import (builtins.fetchTarball {
name = "nixos-unstable";
url = "${json.url}/archive/${json.rev}.tar.gz";
inherit (json) sha256;
});
in
nixpkgs

View File

@@ -27,20 +27,11 @@ load helpers
# Now run inspect locally
run_skopeo inspect dir:$workdir
inspect_local=$output
run_skopeo inspect --raw dir:$workdir
inspect_local_raw=$output
config_digest=$(jq -r '.config.digest' <<<"$inspect_local_raw")
# Each SHA-named layer file (but not the config) must be listed in the output of 'inspect'.
# In all existing versions of Skopeo (with 1.6 being the current as of this comment),
# the output of 'inspect' lists layer digests,
# but not the digest of the config blob ($config_digest), if any.
layers=$(jq -r '.Layers' <<<"$inspect_local")
# Each SHA-named file must be listed in the output of 'inspect'
for sha in $(find $workdir -type f | xargs -l1 basename | egrep '^[0-9a-f]{64}$'); do
if [ "sha256:$sha" != "$config_digest" ]; then
expect_output --from="$layers" --substring "sha256:$sha" \
"Locally-extracted SHA file is present in 'inspect'"
fi
expect_output --from="$inspect_local" --substring "sha256:$sha" \
"Locally-extracted SHA file is present in 'inspect'"
done
# Simple sanity check on 'inspect' output.
@@ -74,47 +65,59 @@ END_EXPECT
}
@test "inspect: env" {
remote_image=docker://quay.io/libpod/fedora:31
remote_image=docker://docker.io/fedora:latest
run_skopeo inspect $remote_image
inspect_remote=$output
# Simple check on 'inspect' output with environment variables.
# 1) Get remote image values of environment variables (the value of 'Env')
# 2) Confirm substring in check_array and the value of 'Env' match.
check_array=(FGC=f31 DISTTAG=f31container)
check_array=(PATH=.* )
remote=$(jq '.Env[]' <<<"$inspect_remote")
for substr in ${check_array[@]}; do
expect_output --from="$remote" --substring "$substr"
done
}
# Tests https://github.com/containers/skopeo/pull/708
@test "inspect: image manifest list w/ diff platform" {
# This image's manifest is for an os + arch that is... um, unlikely
# to support skopeo in the foreseeable future. Or past. The image
# is created by the make-noarch-manifest script in this directory.
img=docker://quay.io/libpod/notmyarch:20210121
# When --raw is provided, can inspect show the raw manifest list, w/o
# requiring any particular platform to be present
# To test whether container image can be inspected successfully w/o
# platform dependency.
# 1) Get current platform arch
# 2) Inspect container image is different from current platform arch
# 3) Compare output w/ expected result
# Get our host arch (what we're running on). This assumes that skopeo
# arch matches podman; it also assumes running podman >= April 2020
# (prior to that, the format keys were lower-case).
arch=$(podman info --format '{{.Host.Arch}}')
# Here we see a revolting workaround for a podman incompatibility
# change: in April 2020, podman info completely changed format
# of the keys. What worked until then now throws an error. We
# need to work with both old and new podman.
arch=$(podman info --format '{{.host.arch}}' || true)
if [[ -z "$arch" ]]; then
arch=$(podman info --format '{{.Host.Arch}}')
fi
# By default, 'inspect' tries to match our host os+arch. This should fail.
run_skopeo 1 inspect $img
expect_output --substring "parsing manifest for image: choosing image instance: no image found in manifest list for architecture $arch, variant " \
"skopeo inspect, without --raw, fails"
case $arch in
"amd64")
diff_arch_list="s390x ppc64le"
;;
"s390x")
diff_arch_list="amd64 ppc64le"
;;
"ppc64le")
diff_arch_list="amd64 s390x"
;;
"*")
diff_arch_list="amd64 s390x ppc64le"
;;
esac
# With --raw, we can inspect
run_skopeo inspect --raw $img
expect_output --substring "manifests.*platform.*architecture" \
"skopeo inspect --raw returns reasonable output"
# ...and what we get should be consistent with what our script created.
archinfo=$(jq -r '.manifests[0].platform | {os,variant,architecture} | join("-")' <<<"$output")
expect_output --from="$archinfo" "amigaos-1000-mc68000" \
"os - variant - architecture of $img"
for arch in $diff_arch_list; do
remote_image=docker://docker.io/$arch/golang
run_skopeo inspect --tls-verify=false --raw $remote_image
remote_arch=$(jq -r '.manifests[0]["platform"]["architecture"]' <<< "$output")
expect_output --from="$remote_arch" "$arch" "platform arch of $remote_image"
done
}
# vim: filetype=sh

View File

@@ -14,7 +14,7 @@ function setup() {
# From remote, to dir1, to local, to dir2;
# compare dir1 and dir2, expect no changes
@test "copy: dir, round trip" {
local remote_image=docker://quay.io/libpod/busybox:latest
local remote_image=docker://docker.io/library/busybox:latest
local localimg=docker://localhost:5000/busybox:unsigned
local dir1=$TESTDIR/dir1
@@ -30,7 +30,7 @@ function setup() {
# Same as above, but using 'oci:' instead of 'dir:' and with a :latest tag
@test "copy: oci, round trip" {
local remote_image=docker://quay.io/libpod/busybox:latest
local remote_image=docker://docker.io/library/busybox:latest
local localimg=docker://localhost:5000/busybox:unsigned
local dir1=$TESTDIR/oci1
@@ -45,8 +45,8 @@ function setup() {
}
# Compression zstd
@test "copy: oci, zstd" {
local remote_image=docker://quay.io/libpod/busybox:latest
@test "copy: oci, round trip, zstd" {
local remote_image=docker://docker.io/library/busybox:latest
local dir=$TESTDIR/dir
@@ -57,17 +57,11 @@ function setup() {
# Check there is at least one file that has the zstd magic number as the first 4 bytes
(for i in $dir/blobs/sha256/*; do test "$(head -c 4 $i)" = $magic && exit 0; done; exit 1)
# Check that the manifest's description of the image's first layer is the zstd layer type
instance=$(jq -r '.manifests[0].digest' $dir/index.json)
[[ "$instance" != null ]]
mediatype=$(jq -r '.layers[0].mediaType' < $dir/blobs/${instance/://})
[[ "$mediatype" == "application/vnd.oci.image.layer.v1.tar+zstd" ]]
}
# Same image, extracted once with :tag and once without
@test "copy: oci w/ and w/o tags" {
local remote_image=docker://quay.io/libpod/busybox:latest
local remote_image=docker://docker.io/library/busybox:latest
local dir1=$TESTDIR/dir1
local dir2=$TESTDIR/dir2
@@ -84,7 +78,7 @@ function setup() {
# Registry -> storage -> oci-archive
@test "copy: registry -> storage -> oci-archive" {
local alpine=quay.io/libpod/alpine:latest
local alpine=docker.io/library/alpine:latest
local tmp=$TESTDIR/oci
run_skopeo copy docker://$alpine containers-storage:$alpine
@@ -100,50 +94,6 @@ function setup() {
docker://localhost:5000/foo
}
# manifest format
@test "copy: manifest format" {
local remote_image=docker://quay.io/libpod/busybox:latest
local dir1=$TESTDIR/dir1
local dir2=$TESTDIR/dir2
run_skopeo copy --format v2s2 $remote_image dir:$dir1
run_skopeo copy --format oci $remote_image dir:$dir2
grep 'application/vnd.docker.distribution.manifest.v2' $dir1/manifest.json
grep 'application/vnd.oci.image' $dir2/manifest.json
}
# additional tag
@test "copy: additional tag" {
local remote_image=docker://quay.io/libpod/busybox:latest
# additional-tag is supported only for docker-archive
run_skopeo copy --additional-tag busybox:mine $remote_image \
docker-archive:$TESTDIR/mybusybox.tar:busybox:latest
mkdir -p $TESTDIR/podmanroot
run podman --root $TESTDIR/podmanroot load -i $TESTDIR/mybusybox.tar
run podman --root $TESTDIR/podmanroot images
expect_output --substring "mine"
}
# shared blob directory
@test "copy: shared blob directory" {
local remote_image=docker://quay.io/libpod/busybox:latest
local shareddir=$TESTDIR/shareddir
local dir1=$TESTDIR/dir1
local dir2=$TESTDIR/dir2
run_skopeo copy --dest-shared-blob-dir $shareddir \
$remote_image oci:$dir1
[ -n "$(ls $shareddir)" ]
[ -z "$(ls $dir1/blobs)" ]
run_skopeo copy --src-shared-blob-dir $shareddir \
oci:$dir1 oci:$dir2
diff -urN $shareddir $dir2/blobs
}
teardown() {
podman rm -f reg

View File

@@ -8,28 +8,19 @@ load helpers
function setup() {
standard_setup
start_registry --with-cert --enable-delete=true reg
start_registry --with-cert reg
}
@test "local registry, with cert" {
# Push to local registry...
run_skopeo copy --dest-cert-dir=$TESTDIR/client-auth \
docker://quay.io/libpod/busybox:latest \
docker://docker.io/library/busybox:latest \
docker://localhost:5000/busybox:unsigned
# ...and pull it back out
run_skopeo copy --src-cert-dir=$TESTDIR/client-auth \
docker://localhost:5000/busybox:unsigned \
dir:$TESTDIR/extracted
# inspect with cert
run_skopeo inspect --cert-dir=$TESTDIR/client-auth \
docker://localhost:5000/busybox:unsigned
expect_output --substring "localhost:5000/busybox"
# delete with cert
run_skopeo delete --cert-dir=$TESTDIR/client-auth \
docker://localhost:5000/busybox:unsigned
}
teardown() {

View File

@@ -18,7 +18,7 @@ function setup() {
testuser=testuser
testpassword=$(random_string 15)
start_registry --testuser=$testuser --testpassword=$testpassword --enable-delete=true reg
start_registry --testuser=$testuser --testpassword=$testpassword reg
}
@test "auth: credentials on command line" {
@@ -43,7 +43,7 @@ function setup() {
# These should pass
run_skopeo copy --dest-tls-verify=false --dcreds=$testuser:$testpassword \
docker://quay.io/libpod/busybox:latest \
docker://docker.io/library/busybox:latest \
docker://localhost:5000/busybox:mine
run_skopeo inspect --tls-verify=false --creds=$testuser:$testpassword \
docker://localhost:5000/busybox:mine
@@ -55,7 +55,7 @@ function setup() {
podman login --tls-verify=false -u $testuser -p $testpassword localhost:5000
run_skopeo copy --dest-tls-verify=false \
docker://quay.io/libpod/busybox:latest \
docker://docker.io/library/busybox:latest \
docker://localhost:5000/busybox:mine
run_skopeo inspect --tls-verify=false docker://localhost:5000/busybox:mine
expect_output --substring "localhost:5000/busybox"
@@ -67,47 +67,6 @@ function setup() {
expect_output --substring "unauthorized: authentication required"
}
@test "auth: copy with --src-creds and --dest-creds" {
run_skopeo copy --dest-tls-verify=false --dest-creds=$testuser:$testpassword \
docker://quay.io/libpod/busybox:latest \
docker://localhost:5000/busybox:mine
run_skopeo copy --src-tls-verify=false --src-creds=$testuser:$testpassword \
docker://localhost:5000/busybox:mine \
dir:$TESTDIR/dir1
run ls $TESTDIR/dir1
expect_output --substring "manifest.json"
}
@test "auth: credentials via authfile" {
podman login --tls-verify=false --authfile $TESTDIR/test.auth -u $testuser -p $testpassword localhost:5000
# copy without authfile: should fail
run_skopeo 1 copy --dest-tls-verify=false \
docker://quay.io/libpod/busybox:latest \
docker://localhost:5000/busybox:mine
# copy with authfile: should work
run_skopeo copy --dest-tls-verify=false \
--authfile $TESTDIR/test.auth \
docker://quay.io/libpod/busybox:latest \
docker://localhost:5000/busybox:mine
# inspect without authfile: should fail
run_skopeo 1 inspect --tls-verify=false docker://localhost:5000/busybox:mine
expect_output --substring "unauthorized: authentication required"
# inspect with authfile: should work
run_skopeo inspect --tls-verify=false --authfile $TESTDIR/test.auth docker://localhost:5000/busybox:mine
expect_output --substring "localhost:5000/busybox"
# delete without authfile: should fail
run_skopeo 1 delete --tls-verify=false docker://localhost:5000/busybox:mine
expect_output --substring "authentication required"
# delete with authfile: should work
run_skopeo delete --tls-verify=false --authfile $TESTDIR/test.auth docker://localhost:5000/busybox:mine
}
teardown() {
podman rm -f reg

View File

@@ -92,7 +92,7 @@ END_POLICY_JSON
fi
# Cache local copy
run_skopeo copy docker://quay.io/libpod/busybox:latest \
run_skopeo copy docker://docker.io/library/busybox:latest \
dir:$TESTDIR/busybox
# Push a bunch of images. Do so *without* --policy flag; this lets us
@@ -143,75 +143,6 @@ END_PUSH
END_TESTS
}
@test "signing: remove signature" {
run_skopeo '?' standalone-sign /dev/null busybox alice@test.redhat.com -o /dev/null
if [[ "$output" =~ 'signing is not supported' ]]; then
skip "skopeo built without support for creating signatures"
return 1
fi
if [ "$status" -ne 0 ]; then
die "exit code is $status; expected 0"
fi
# Cache local copy
run_skopeo copy docker://quay.io/libpod/busybox:latest \
dir:$TESTDIR/busybox
# Push a signed image
run_skopeo --registries.d $REGISTRIES_D \
copy --dest-tls-verify=false \
--sign-by=alice@test.redhat.com \
dir:$TESTDIR/busybox \
docker://localhost:5000/myns/alice:signed
# Fetch the image with signature
run_skopeo --registries.d $REGISTRIES_D \
--policy $POLICY_JSON \
copy --src-tls-verify=false \
docker://localhost:5000/myns/alice:signed \
dir:$TESTDIR/busybox-signed
# Fetch the image with removing signature
run_skopeo --registries.d $REGISTRIES_D \
--policy $POLICY_JSON \
copy --src-tls-verify=false \
--remove-signatures \
docker://localhost:5000/myns/alice:signed \
dir:$TESTDIR/busybox-unsigned
ls $TESTDIR/busybox-signed | grep "signature"
[ -z "$(ls $TESTDIR/busybox-unsigned | grep "signature")" ]
}
@test "signing: standalone" {
run_skopeo '?' standalone-sign /dev/null busybox alice@test.redhat.com -o /dev/null
if [[ "$output" =~ 'signing is not supported' ]]; then
skip "skopeo built without support for creating signatures"
return 1
fi
if [ "$status" -ne 0 ]; then
die "exit code is $status; expected 0"
fi
run_skopeo copy --dest-tls-verify=false \
docker://quay.io/libpod/busybox:latest \
docker://localhost:5000/busybox:latest
run_skopeo copy --src-tls-verify=false \
docker://localhost:5000/busybox:latest \
dir:$TESTDIR/busybox
# Standalone sign
run_skopeo standalone-sign -o $TESTDIR/busybox.signature \
$TESTDIR/busybox/manifest.json \
localhost:5000/busybox:latest \
alice@test.redhat.com
# Standalone verify
fingerprint=$(gpg --list-keys | grep -B1 alice.test.redhat.com | head -n 1)
run_skopeo standalone-verify $TESTDIR/busybox/manifest.json \
localhost:5000/busybox:latest \
$fingerprint \
$TESTDIR/busybox.signature
# manifest digest
digest=$(echo "$output" | awk '{print $4;}')
run_skopeo manifest-digest $TESTDIR/busybox/manifest.json
expect_output $digest
}
teardown() {
podman rm -f reg

View File

@@ -13,7 +13,7 @@ function setup() {
# delete image from registry
@test "delete: remove image from registry" {
local remote_image=docker://quay.io/libpod/busybox:latest
local remote_image=docker://docker.io/library/busybox:latest
local localimg=docker://localhost:5000/busybox:unsigned
local output=

View File

@@ -6,7 +6,7 @@ SKOPEO_BINARY=${SKOPEO_BINARY:-$(dirname ${BASH_SOURCE})/../skopeo}
SKOPEO_TIMEOUT=${SKOPEO_TIMEOUT:-300}
# Default image to run as a local registry
REGISTRY_FQIN=${SKOPEO_TEST_REGISTRY_FQIN:-quay.io/libpod/registry:2}
REGISTRY_FQIN=${SKOPEO_TEST_REGISTRY_FQIN:-docker.io/library/registry:2}
###############################################################################
# BEGIN setup/teardown
@@ -194,7 +194,7 @@ function expect_output() {
fi
# This is a multi-line message, which may in turn contain multi-line
# output, so let's format it ourselves, readably
# output, so let's format it ourself, readably
local -a actual_split
readarray -t actual_split <<<"$actual"
printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2
@@ -331,8 +331,7 @@ start_registry() {
log_and_run openssl req -newkey rsa:4096 -nodes -sha256 \
-keyout $AUTHDIR/domain.key -x509 -days 2 \
-out $CERT \
-subj "/C=US/ST=Foo/L=Bar/O=Red Hat, Inc./CN=registry host certificate" \
-addext subjectAltName=DNS:localhost
-subj "/C=US/ST=Foo/L=Bar/O=Red Hat, Inc./CN=localhost"
fi
reg_args+=(
@@ -356,7 +355,7 @@ start_registry() {
return
fi
timeout=$(( timeout - 1 ))
timeout=$(expr $timeout - 1)
sleep 1
done
die "Timed out waiting for registry container to respond on :$port"

View File

@@ -1,70 +0,0 @@
#!/bin/sh
#
# Tool for creating an image whose OS and arch will (probably) never
# match a system on which skopeo will run. This image will be used
# in the 'inspect' test.
#
set -ex
# Name and tag of the image we create
imgname=notmyarch
imgtag=$(date +%Y%m%d)
# (In case older image exists from a prior run)
buildah rmi $imgname:$imgtag &>/dev/null || true
#
# Step 1: create an image containing only a README and a copy of this script
#
id=$(buildah from scratch)
now=$(date --rfc-3339=seconds)
readme=$(mktemp -t README.XXXXXXXX)
ME=$(basename $0)
cat >| $readme <<EOF
This is a dummy image intended solely for skopeo testing.
This image was created $now
The script used to create this image is available as $ME
EOF
buildah copy $id $readme /README
buildah copy $id $0 /$ME
buildah commit $id my_tmp_image
buildah rm $id
#
# Step 2: create a manifest list, then add the above image but with
# an os+arch override.
#
buildah manifest create $imgname:$imgtag
buildah manifest add \
--os amigaos \
--arch mc68000 \
--variant 1000 \
$imgname:$imgtag my_tmp_image
# Done. Show instructions.
cat <<EOF
DONE!
You can inspect the created image with:
skopeo inspect --raw containers-storage:localhost/$imgname:$imgtag | jq .
(FIXME: is there a way to, like, mount the image and verify the files?)
If you're happy with this image, you can now:
buildah manifest push --all $imgname:$imgtag docker://quay.io/libpod/$imgname:$imgtag
Once done, you urgently need to:
buildah rmi $imgname:$imgtag my_tmp_image
If you don't do this, 'podman images' will barf catastrophically!
EOF

View File

@@ -1,2 +1,5 @@
TAGS
tags
.*.swp
tomlcheck/tomlcheck
toml.test
/toml-test

15
vendor/github.com/BurntSushi/toml/.travis.yml generated vendored Normal file
View File

@@ -0,0 +1,15 @@
language: go
go:
- 1.1
- 1.2
- 1.3
- 1.4
- 1.5
- 1.6
- tip
install:
- go install ./...
- go get github.com/BurntSushi/toml-test
script:
- export PATH="$PATH:$HOME/gopath/bin"
- make test

View File

@@ -1 +1,3 @@
Compatible with TOML version [v1.0.0](https://toml.io/en/v1.0.0).
Compatible with TOML version
[v0.4.0](https://github.com/toml-lang/toml/blob/v0.4.0/versions/en/toml-v0.4.0.md)

19
vendor/github.com/BurntSushi/toml/Makefile generated vendored Normal file
View File

@@ -0,0 +1,19 @@
install:
go install ./...
test: install
go test -v
toml-test toml-test-decoder
toml-test -encoder toml-test-encoder
fmt:
gofmt -w *.go */*.go
colcheck *.go */*.go
tags:
find ./ -name '*.go' -print0 | xargs -0 gotags > TAGS
push:
git push origin master
git push github master

View File

@@ -6,22 +6,27 @@ packages. This package also supports the `encoding.TextUnmarshaler` and
`encoding.TextMarshaler` interfaces so that you can define custom data
representations. (There is an example of this below.)
Compatible with TOML version [v1.0.0](https://toml.io/en/v1.0.0).
Spec: https://github.com/toml-lang/toml
Documentation: https://godocs.io/github.com/BurntSushi/toml
Compatible with TOML version
[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md)
See the [releases page](https://github.com/BurntSushi/toml/releases) for a
changelog; this information is also in the git tag annotations (e.g. `git show
v0.4.0`).
Documentation: https://godoc.org/github.com/BurntSushi/toml
This library requires Go 1.13 or newer; install it with:
Installation:
$ go get github.com/BurntSushi/toml
```bash
go get github.com/BurntSushi/toml
```
It also comes with a TOML validator CLI tool:
Try the toml validator:
$ go get github.com/BurntSushi/toml/cmd/tomlv
$ tomlv some-toml-file.toml
```bash
go get github.com/BurntSushi/toml/cmd/tomlv
tomlv some-toml-file.toml
```
[![Build Status](https://travis-ci.org/BurntSushi/toml.svg?branch=master)](https://travis-ci.org/BurntSushi/toml) [![GoDoc](https://godoc.org/github.com/BurntSushi/toml?status.svg)](https://godoc.org/github.com/BurntSushi/toml)
### Testing
@@ -31,8 +36,8 @@ and the encoder.
### Examples
This package works similarly to how the Go standard library handles XML and
JSON. Namely, data is loaded into Go values via reflection.
This package works similarly to how the Go standard library handles `XML`
and `JSON`. Namely, data is loaded into Go values via reflection.
For the simplest example, consider some TOML file as just a list of keys
and values:
@@ -49,11 +54,11 @@ Which could be defined in Go as:
```go
type Config struct {
Age int
Cats []string
Pi float64
Perfection []int
DOB time.Time // requires `import time`
Age int
Cats []string
Pi float64
Perfection []int
DOB time.Time // requires `import time`
}
```
@@ -79,9 +84,6 @@ type TOML struct {
}
```
Beware that like other most other decoders **only exported fields** are
considered when encoding and decoding; private fields are silently ignored.
### Using the `encoding.TextUnmarshaler` interface
Here's an example that automatically parses duration strings into
@@ -101,19 +103,19 @@ Which can be decoded with:
```go
type song struct {
Name string
Duration duration
Name string
Duration duration
}
type songs struct {
Song []song
Song []song
}
var favorites songs
if _, err := toml.Decode(blob, &favorites); err != nil {
log.Fatal(err)
log.Fatal(err)
}
for _, s := range favorites.Song {
fmt.Printf("%s (%s)\n", s.Name, s.Duration)
fmt.Printf("%s (%s)\n", s.Name, s.Duration)
}
```
@@ -132,9 +134,6 @@ func (d *duration) UnmarshalText(text []byte) error {
}
```
To target TOML specifically you can implement `UnmarshalTOML` TOML interface in
a similar way.
### More complex usage
Here's an example of how to load the example from the official spec page:
@@ -181,23 +180,23 @@ And the corresponding Go types are:
```go
type tomlConfig struct {
Title string
Owner ownerInfo
DB database `toml:"database"`
Title string
Owner ownerInfo
DB database `toml:"database"`
Servers map[string]server
Clients clients
}
type ownerInfo struct {
Name string
Org string `toml:"organization"`
Bio string
DOB time.Time
Org string `toml:"organization"`
Bio string
DOB time.Time
}
type database struct {
Server string
Ports []int
Server string
Ports []int
ConnMax int `toml:"connection_max"`
Enabled bool
}
@@ -208,7 +207,7 @@ type server struct {
}
type clients struct {
Data [][]interface{}
Data [][]interface{}
Hosts []string
}
```
@@ -217,4 +216,3 @@ Note that a case insensitive match will be tried if an exact match can't be
found.
A working example of the above can be found in `_examples/example.{go,toml}`.

View File

@@ -1,17 +1,19 @@
package toml
import (
"encoding"
"fmt"
"io"
"io/ioutil"
"math"
"os"
"reflect"
"strings"
"time"
)
func e(format string, args ...interface{}) error {
return fmt.Errorf("toml: "+format, args...)
}
// Unmarshaler is the interface implemented by objects that can unmarshal a
// TOML description of themselves.
type Unmarshaler interface {
@@ -25,21 +27,30 @@ func Unmarshal(p []byte, v interface{}) error {
}
// Primitive is a TOML value that hasn't been decoded into a Go value.
// When using the various `Decode*` functions, the type `Primitive` may
// be given to any value, and its decoding will be delayed.
//
// This type can be used for any value, which will cause decoding to be delayed.
// You can use the PrimitiveDecode() function to "manually" decode these values.
// A `Primitive` value can be decoded using the `PrimitiveDecode` function.
//
// NOTE: The underlying representation of a `Primitive` value is subject to
// change. Do not rely on it.
// The underlying representation of a `Primitive` value is subject to change.
// Do not rely on it.
//
// NOTE: Primitive values are still parsed, so using them will only avoid the
// overhead of reflection. They can be useful when you don't know the exact type
// of TOML data until runtime.
// N.B. Primitive values are still parsed, so using them will only avoid
// the overhead of reflection. They can be useful when you don't know the
// exact type of TOML data until run time.
type Primitive struct {
undecoded interface{}
context Key
}
// DEPRECATED!
//
// Use MetaData.PrimitiveDecode instead.
func PrimitiveDecode(primValue Primitive, v interface{}) error {
md := MetaData{decoded: make(map[string]bool)}
return md.unify(primValue.undecoded, rvalue(v))
}
// PrimitiveDecode is just like the other `Decode*` functions, except it
// decodes a TOML value that has already been parsed. Valid primitive values
// can *only* be obtained from values filled by the decoder functions,
@@ -57,51 +68,43 @@ func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error {
return md.unify(primValue.undecoded, rvalue(v))
}
// Decoder decodes TOML data.
// Decode will decode the contents of `data` in TOML format into a pointer
// `v`.
//
// TOML tables correspond to Go structs or maps (dealer's choice they can be
// used interchangeably).
// TOML hashes correspond to Go structs or maps. (Dealer's choice. They can be
// used interchangeably.)
//
// TOML table arrays correspond to either a slice of structs or a slice of maps.
// TOML arrays of tables correspond to either a slice of structs or a slice
// of maps.
//
// TOML datetimes correspond to Go time.Time values. Local datetimes are parsed
// in the local timezone.
// TOML datetimes correspond to Go `time.Time` values.
//
// All other TOML types (float, string, int, bool and array) correspond to the
// obvious Go types.
// All other TOML types (float, string, int, bool and array) correspond
// to the obvious Go types.
//
// An exception to the above rules is if a type implements the TextUnmarshaler
// interface, in which case any primitive TOML value (floats, strings, integers,
// booleans, datetimes) will be converted to a []byte and given to the value's
// UnmarshalText method. See the Unmarshaler example for a demonstration with
// time duration strings.
// An exception to the above rules is if a type implements the
// encoding.TextUnmarshaler interface. In this case, any primitive TOML value
// (floats, strings, integers, booleans and datetimes) will be converted to
// a byte string and given to the value's UnmarshalText method. See the
// Unmarshaler example for a demonstration with time duration strings.
//
// Key mapping
//
// TOML keys can map to either keys in a Go map or field names in a Go struct.
// The special `toml` struct tag can be used to map TOML keys to struct fields
// that don't match the key name exactly (see the example). A case insensitive
// match to struct names will be tried if an exact match can't be found.
// TOML keys can map to either keys in a Go map or field names in a Go
// struct. The special `toml` struct tag may be used to map TOML keys to
// struct fields that don't match the key name exactly. (See the example.)
// A case insensitive match to struct names will be tried if an exact match
// can't be found.
//
// The mapping between TOML values and Go values is loose. That is, there may
// exist TOML values that cannot be placed into your representation, and there
// may be parts of your representation that do not correspond to TOML values.
// This loose mapping can be made stricter by using the IsDefined and/or
// Undecoded methods on the MetaData returned.
// The mapping between TOML values and Go values is loose. That is, there
// may exist TOML values that cannot be placed into your representation, and
// there may be parts of your representation that do not correspond to
// TOML values. This loose mapping can be made stricter by using the IsDefined
// and/or Undecoded methods on the MetaData returned.
//
// This decoder does not handle cyclic types. Decode will not terminate if a
// cyclic type is passed.
type Decoder struct {
r io.Reader
}
// NewDecoder creates a new Decoder.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r: r}
}
// Decode TOML data in to the pointer `v`.
func (dec *Decoder) Decode(v interface{}) (MetaData, error) {
// This decoder will not handle cyclic types. If a cyclic type is passed,
// `Decode` will not terminate.
func Decode(data string, v interface{}) (MetaData, error) {
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Ptr {
return MetaData{}, e("Decode of non-pointer %s", reflect.TypeOf(v))
@@ -109,15 +112,7 @@ func (dec *Decoder) Decode(v interface{}) (MetaData, error) {
if rv.IsNil() {
return MetaData{}, e("Decode of nil %s", reflect.TypeOf(v))
}
// TODO: have parser should read from io.Reader? Or at the very least, make
// it read from []byte rather than string
data, err := ioutil.ReadAll(dec.r)
if err != nil {
return MetaData{}, err
}
p, err := parse(string(data))
p, err := parse(data)
if err != nil {
return MetaData{}, err
}
@@ -128,22 +123,24 @@ func (dec *Decoder) Decode(v interface{}) (MetaData, error) {
return md, md.unify(p.mapping, indirect(rv))
}
// Decode the TOML data in to the pointer v.
//
// See the documentation on Decoder for a description of the decoding process.
func Decode(data string, v interface{}) (MetaData, error) {
return NewDecoder(strings.NewReader(data)).Decode(v)
}
// DecodeFile is just like Decode, except it will automatically read the
// contents of the file at path and decode it for you.
func DecodeFile(path string, v interface{}) (MetaData, error) {
fp, err := os.Open(path)
// contents of the file at `fpath` and decode it for you.
func DecodeFile(fpath string, v interface{}) (MetaData, error) {
bs, err := ioutil.ReadFile(fpath)
if err != nil {
return MetaData{}, err
}
defer fp.Close()
return NewDecoder(fp).Decode(v)
return Decode(string(bs), v)
}
// DecodeReader is just like Decode, except it will consume all bytes
// from the reader and decode it for you.
func DecodeReader(r io.Reader, v interface{}) (MetaData, error) {
bs, err := ioutil.ReadAll(r)
if err != nil {
return MetaData{}, err
}
return Decode(string(bs), v)
}
// unify performs a sort of type unification based on the structure of `rv`,
@@ -152,8 +149,8 @@ func DecodeFile(path string, v interface{}) (MetaData, error) {
// Any type mismatch produces an error. Finding a type that we don't know
// how to handle produces an unsupported type error.
func (md *MetaData) unify(data interface{}, rv reflect.Value) error {
// Special case. Look for a `Primitive` value.
// TODO: #76 would make this superfluous after implemented.
if rv.Type() == reflect.TypeOf((*Primitive)(nil)).Elem() {
// Save the undecoded data and the key context into the primitive
// value.
@@ -173,17 +170,25 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error {
}
}
// Special case. Handle time.Time values specifically.
// TODO: Remove this code when we decide to drop support for Go 1.1.
// This isn't necessary in Go 1.2 because time.Time satisfies the encoding
// interfaces.
if rv.Type().AssignableTo(rvalue(time.Time{}).Type()) {
return md.unifyDatetime(data, rv)
}
// Special case. Look for a value satisfying the TextUnmarshaler interface.
if v, ok := rv.Interface().(encoding.TextUnmarshaler); ok {
if v, ok := rv.Interface().(TextUnmarshaler); ok {
return md.unifyText(data, v)
}
// TODO:
// BUG(burntsushi)
// The behavior here is incorrect whenever a Go type satisfies the
// encoding.TextUnmarshaler interface but also corresponds to a TOML hash or
// array. In particular, the unmarshaler should only be applied to primitive
// TOML values. But at this point, it will be applied to all kinds of values
// and produce an incorrect error whenever those values are hashes or arrays
// (including arrays of tables).
// encoding.TextUnmarshaler interface but also corresponds to a TOML
// hash or array. In particular, the unmarshaler should only be applied
// to primitive TOML values. But at this point, it will be applied to
// all kinds of values and produce an incorrect error whenever those values
// are hashes or arrays (including arrays of tables).
k := rv.Kind()
@@ -272,12 +277,6 @@ func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error {
}
func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error {
if k := rv.Type().Key().Kind(); k != reflect.String {
return fmt.Errorf(
"toml: cannot decode to a map with non-string key type (%s in %q)",
k, rv.Type())
}
tmap, ok := mapping.(map[string]interface{})
if !ok {
if tmap == nil {
@@ -313,8 +312,10 @@ func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error {
}
return badtype("slice", data)
}
if l := datav.Len(); l != rv.Len() {
return e("expected array length %d; got TOML array of length %d", rv.Len(), l)
sliceLen := datav.Len()
if sliceLen != rv.Len() {
return e("expected array length %d; got TOML array of length %d",
rv.Len(), sliceLen)
}
return md.unifySliceArray(datav, rv)
}
@@ -336,10 +337,11 @@ func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error {
}
func (md *MetaData) unifySliceArray(data, rv reflect.Value) error {
l := data.Len()
for i := 0; i < l; i++ {
err := md.unify(data.Index(i).Interface(), indirect(rv.Index(i)))
if err != nil {
sliceLen := data.Len()
for i := 0; i < sliceLen; i++ {
v := data.Index(i).Interface()
sliceval := indirect(rv.Index(i))
if err := md.unify(v, sliceval); err != nil {
return err
}
}
@@ -437,7 +439,7 @@ func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error {
return nil
}
func (md *MetaData) unifyText(data interface{}, v encoding.TextUnmarshaler) error {
func (md *MetaData) unifyText(data interface{}, v TextUnmarshaler) error {
var s string
switch sdata := data.(type) {
case TextMarshaler:
@@ -480,7 +482,7 @@ func indirect(v reflect.Value) reflect.Value {
if v.Kind() != reflect.Ptr {
if v.CanSet() {
pv := v.Addr()
if _, ok := pv.Interface().(encoding.TextUnmarshaler); ok {
if _, ok := pv.Interface().(TextUnmarshaler); ok {
return pv
}
}
@@ -496,16 +498,12 @@ func isUnifiable(rv reflect.Value) bool {
if rv.CanSet() {
return true
}
if _, ok := rv.Interface().(encoding.TextUnmarshaler); ok {
if _, ok := rv.Interface().(TextUnmarshaler); ok {
return true
}
return false
}
func e(format string, args ...interface{}) error {
return fmt.Errorf("toml: "+format, args...)
}
func badtype(expected string, data interface{}) error {
return e("cannot load TOML value of type %T into a Go %s", data, expected)
}

View File

@@ -1,18 +0,0 @@
// +build go1.16
package toml
import (
"io/fs"
)
// DecodeFS is just like Decode, except it will automatically read the contents
// of the file at `path` from a fs.FS instance.
func DecodeFS(fsys fs.FS, path string, v interface{}) (MetaData, error) {
fp, err := fsys.Open(path)
if err != nil {
return MetaData{}, err
}
defer fp.Close()
return NewDecoder(fp).Decode(v)
}

View File

@@ -2,9 +2,9 @@ package toml
import "strings"
// MetaData allows access to meta information about TOML data that may not be
// inferable via reflection. In particular, whether a key has been defined and
// the TOML type of a key.
// MetaData allows access to meta information about TOML data that may not
// be inferrable via reflection. In particular, whether a key has been defined
// and the TOML type of a key.
type MetaData struct {
mapping map[string]interface{}
types map[string]tomlType
@@ -13,11 +13,10 @@ type MetaData struct {
context Key // Used only during decoding.
}
// IsDefined reports if the key exists in the TOML data.
//
// The key should be specified hierarchically, for example to access the TOML
// key "a.b.c" you would use:
// IsDefined returns true if the key given exists in the TOML data. The key
// should be specified hierarchially. e.g.,
//
// // access the TOML key 'a.b.c'
// IsDefined("a", "b", "c")
//
// IsDefined will return false if an empty key given. Keys are case sensitive.
@@ -42,8 +41,8 @@ func (md *MetaData) IsDefined(key ...string) bool {
// Type returns a string representation of the type of the key specified.
//
// Type will return the empty string if given an empty key or a key that does
// not exist. Keys are case sensitive.
// Type will return the empty string if given an empty key or a key that
// does not exist. Keys are case sensitive.
func (md *MetaData) Type(key ...string) string {
fullkey := strings.Join(key, ".")
if typ, ok := md.types[fullkey]; ok {
@@ -52,11 +51,13 @@ func (md *MetaData) Type(key ...string) string {
return ""
}
// Key represents any TOML key, including key groups. Use (MetaData).Keys to get
// values of this type.
// Key is the type of any TOML key, including key groups. Use (MetaData).Keys
// to get values of this type.
type Key []string
func (k Key) String() string { return strings.Join(k, ".") }
func (k Key) String() string {
return strings.Join(k, ".")
}
func (k Key) maybeQuotedAll() string {
var ss []string
@@ -67,9 +68,6 @@ func (k Key) maybeQuotedAll() string {
}
func (k Key) maybeQuoted(i int) string {
if k[i] == "" {
return `""`
}
quote := false
for _, c := range k[i] {
if !isBareKeyChar(c) {
@@ -78,7 +76,7 @@ func (k Key) maybeQuoted(i int) string {
}
}
if quote {
return `"` + quotedReplacer.Replace(k[i]) + `"`
return "\"" + strings.Replace(k[i], "\"", "\\\"", -1) + "\""
}
return k[i]
}
@@ -91,10 +89,10 @@ func (k Key) add(piece string) Key {
}
// Keys returns a slice of every key in the TOML data, including key groups.
//
// Each key is itself a slice, where the first element is the top of the
// hierarchy and the last is the most specific. The list will have the same
// order as the keys appeared in the TOML data.
// hierarchy and the last is the most specific.
//
// The list will have the same order as the keys appeared in the TOML data.
//
// All keys returned are non-empty.
func (md *MetaData) Keys() []Key {

View File

@@ -1,33 +0,0 @@
package toml
import (
"encoding"
"io"
)
// DEPRECATED!
//
// Use the identical encoding.TextMarshaler instead. It is defined here to
// support Go 1.1 and older.
type TextMarshaler encoding.TextMarshaler
// DEPRECATED!
//
// Use the identical encoding.TextUnmarshaler instead. It is defined here to
// support Go 1.1 and older.
type TextUnmarshaler encoding.TextUnmarshaler
// DEPRECATED!
//
// Use MetaData.PrimitiveDecode instead.
func PrimitiveDecode(primValue Primitive, v interface{}) error {
md := MetaData{decoded: make(map[string]bool)}
return md.unify(primValue.undecoded, rvalue(v))
}
// DEPRECATED!
//
// Use NewDecoder(reader).Decode(&v) instead.
func DecodeReader(r io.Reader, v interface{}) (MetaData, error) {
return NewDecoder(r).Decode(v)
}

View File

@@ -1,13 +1,27 @@
/*
Package toml implements decoding and encoding of TOML files.
Package toml provides facilities for decoding and encoding TOML configuration
files via reflection. There is also support for delaying decoding with
the Primitive type, and querying the set of keys in a TOML document with the
MetaData type.
This package supports TOML v1.0.0, as listed on https://toml.io
The specification implemented: https://github.com/toml-lang/toml
There is also support for delaying decoding with the Primitive type, and
querying the set of keys in a TOML document with the MetaData type.
The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify
whether a file is a valid TOML document. It can also be used to print the
type of each key in a TOML document.
The github.com/BurntSushi/toml/cmd/tomlv package implements a TOML validator,
and can be used to verify if TOML document is valid. It can also be used to
print the type of each key.
Testing
There are two important types of tests used for this package. The first is
contained inside '*_test.go' files and uses the standard Go unit testing
framework. These tests are primarily devoted to holistically testing the
decoder and encoder.
The second type of testing is used to verify the implementation's adherence
to the TOML specification. These tests have been factored into their own
project: https://github.com/BurntSushi/toml-test
The reason the tests are in a separate project is so that they can be used by
any implementation of TOML. Namely, it is language agnostic.
*/
package toml

View File

@@ -2,92 +2,48 @@ package toml
import (
"bufio"
"encoding"
"errors"
"fmt"
"io"
"math"
"reflect"
"sort"
"strconv"
"strings"
"time"
"github.com/BurntSushi/toml/internal"
)
type tomlEncodeError struct{ error }
var (
errArrayNilElement = errors.New("toml: cannot encode array with nil element")
errNonString = errors.New("toml: cannot encode a map with non-string key type")
errAnonNonStruct = errors.New("toml: cannot encode an anonymous field that is not a struct")
errNoKey = errors.New("toml: top-level values must be Go maps or structs")
errAnything = errors.New("") // used in testing
errArrayMixedElementTypes = errors.New(
"toml: cannot encode array with mixed element types")
errArrayNilElement = errors.New(
"toml: cannot encode array with nil element")
errNonString = errors.New(
"toml: cannot encode a map with non-string key type")
errAnonNonStruct = errors.New(
"toml: cannot encode an anonymous field that is not a struct")
errArrayNoTable = errors.New(
"toml: TOML array element cannot contain a table")
errNoKey = errors.New(
"toml: top-level values must be Go maps or structs")
errAnything = errors.New("") // used in testing
)
var quotedReplacer = strings.NewReplacer(
"\t", "\\t",
"\n", "\\n",
"\r", "\\r",
"\"", "\\\"",
"\\", "\\\\",
"\x00", `\u0000`,
"\x01", `\u0001`,
"\x02", `\u0002`,
"\x03", `\u0003`,
"\x04", `\u0004`,
"\x05", `\u0005`,
"\x06", `\u0006`,
"\x07", `\u0007`,
"\b", `\b`,
"\t", `\t`,
"\n", `\n`,
"\x0b", `\u000b`,
"\f", `\f`,
"\r", `\r`,
"\x0e", `\u000e`,
"\x0f", `\u000f`,
"\x10", `\u0010`,
"\x11", `\u0011`,
"\x12", `\u0012`,
"\x13", `\u0013`,
"\x14", `\u0014`,
"\x15", `\u0015`,
"\x16", `\u0016`,
"\x17", `\u0017`,
"\x18", `\u0018`,
"\x19", `\u0019`,
"\x1a", `\u001a`,
"\x1b", `\u001b`,
"\x1c", `\u001c`,
"\x1d", `\u001d`,
"\x1e", `\u001e`,
"\x1f", `\u001f`,
"\x7f", `\u007f`,
)
// Encoder encodes a Go to a TOML document.
// Encoder controls the encoding of Go values to a TOML document to some
// io.Writer.
//
// The mapping between Go values and TOML values should be precisely the same as
// for the Decode* functions. Similarly, the TextMarshaler interface is
// supported by encoding the resulting bytes as strings. If you want to write
// arbitrary binary data then you will need to use something like base64 since
// TOML does not have any binary types.
//
// When encoding TOML hashes (Go maps or structs), keys without any sub-hashes
// are encoded first.
//
// Go maps will be sorted alphabetically by key for deterministic output.
//
// Encoding Go values without a corresponding TOML representation will return an
// error. Examples of this includes maps with non-string keys, slices with nil
// elements, embedded non-struct types, and nested slices containing maps or
// structs. (e.g. [][]map[string]string is not allowed but []map[string]string
// is okay, as is []map[string][]string).
//
// NOTE: Only exported keys are encoded due to the use of reflection. Unexported
// keys are silently discarded.
// The indentation level can be controlled with the Indent field.
type Encoder struct {
// The string to use for a single indentation level. The default is two
// spaces.
// A single indentation level. By default it is two spaces.
Indent string
// hasWritten is whether we have written any output to w yet.
@@ -95,7 +51,8 @@ type Encoder struct {
w *bufio.Writer
}
// NewEncoder create a new Encoder.
// NewEncoder returns a TOML encoder that encodes Go values to the io.Writer
// given. By default, a single indentation level is 2 spaces.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{
w: bufio.NewWriter(w),
@@ -103,10 +60,29 @@ func NewEncoder(w io.Writer) *Encoder {
}
}
// Encode writes a TOML representation of the Go value to the Encoder's writer.
// Encode writes a TOML representation of the Go value to the underlying
// io.Writer. If the value given cannot be encoded to a valid TOML document,
// then an error is returned.
//
// An error is returned if the value given cannot be encoded to a valid TOML
// document.
// The mapping between Go values and TOML values should be precisely the same
// as for the Decode* functions. Similarly, the TextMarshaler interface is
// supported by encoding the resulting bytes as strings. (If you want to write
// arbitrary binary data then you will need to use something like base64 since
// TOML does not have any binary types.)
//
// When encoding TOML hashes (i.e., Go maps or structs), keys without any
// sub-hashes are encoded first.
//
// If a Go map is encoded, then its keys are sorted alphabetically for
// deterministic output. More control over this behavior may be provided if
// there is demand for it.
//
// Encoding Go values without a corresponding TOML representation---like map
// types with non-string keys---will cause an error to be returned. Similarly
// for mixed arrays/slices, arrays/slices with nil elements, embedded
// non-struct types and nested slices containing maps or structs.
// (e.g., [][]map[string]string is not allowed but []map[string]string is OK
// and so is []map[string][]string.)
func (enc *Encoder) Encode(v interface{}) error {
rv := eindirect(reflect.ValueOf(v))
if err := enc.safeEncode(Key([]string{}), rv); err != nil {
@@ -134,13 +110,9 @@ func (enc *Encoder) encode(key Key, rv reflect.Value) {
// Special case. If we can marshal the type to text, then we used that.
// Basically, this prevents the encoder for handling these types as
// generic structs (or whatever the underlying type of a TextMarshaler is).
switch t := rv.Interface().(type) {
case time.Time, encoding.TextMarshaler:
enc.writeKeyValue(key, rv, false)
return
// TODO: #76 would make this superfluous after implemented.
case Primitive:
enc.encode(key, reflect.ValueOf(t.undecoded))
switch rv.Interface().(type) {
case time.Time, TextMarshaler:
enc.keyEqElement(key, rv)
return
}
@@ -151,12 +123,12 @@ func (enc *Encoder) encode(key Key, rv reflect.Value) {
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32,
reflect.Uint64,
reflect.Float32, reflect.Float64, reflect.String, reflect.Bool:
enc.writeKeyValue(key, rv, false)
enc.keyEqElement(key, rv)
case reflect.Array, reflect.Slice:
if typeEqual(tomlArrayHash, tomlTypeOfGo(rv)) {
enc.eArrayOfTables(key, rv)
} else {
enc.writeKeyValue(key, rv, false)
enc.keyEqElement(key, rv)
}
case reflect.Interface:
if rv.IsNil() {
@@ -176,32 +148,22 @@ func (enc *Encoder) encode(key Key, rv reflect.Value) {
case reflect.Struct:
enc.eTable(key, rv)
default:
encPanic(fmt.Errorf("unsupported type for key '%s': %s", key, k))
panic(e("unsupported type for key '%s': %s", key, k))
}
}
// eElement encodes any value that can be an array element.
// eElement encodes any value that can be an array element (primitives and
// arrays).
func (enc *Encoder) eElement(rv reflect.Value) {
switch v := rv.Interface().(type) {
case time.Time: // Using TextMarshaler adds extra quotes, which we don't want.
format := time.RFC3339Nano
switch v.Location() {
case internal.LocalDatetime:
format = "2006-01-02T15:04:05.999999999"
case internal.LocalDate:
format = "2006-01-02"
case internal.LocalTime:
format = "15:04:05.999999999"
}
switch v.Location() {
default:
enc.wf(v.Format(format))
case internal.LocalDatetime, internal.LocalDate, internal.LocalTime:
enc.wf(v.In(time.UTC).Format(format))
}
case time.Time:
// Special case time.Time as a primitive. Has to come before
// TextMarshaler below because time.Time implements
// encoding.TextMarshaler, but we need to always use UTC.
enc.wf(v.UTC().Format("2006-01-02T15:04:05Z"))
return
case encoding.TextMarshaler:
// Use text marshaler if it's available for this value.
case TextMarshaler:
// Special case. Use text marshaler if it's available for this value.
if s, err := v.MarshalText(); err != nil {
encPanic(err)
} else {
@@ -209,49 +171,32 @@ func (enc *Encoder) eElement(rv reflect.Value) {
}
return
}
switch rv.Kind() {
case reflect.String:
enc.writeQuoted(rv.String())
case reflect.Bool:
enc.wf(strconv.FormatBool(rv.Bool()))
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
reflect.Int64:
enc.wf(strconv.FormatInt(rv.Int(), 10))
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
case reflect.Uint, reflect.Uint8, reflect.Uint16,
reflect.Uint32, reflect.Uint64:
enc.wf(strconv.FormatUint(rv.Uint(), 10))
case reflect.Float32:
f := rv.Float()
if math.IsNaN(f) {
enc.wf("nan")
} else if math.IsInf(f, 0) {
enc.wf("%cinf", map[bool]byte{true: '-', false: '+'}[math.Signbit(f)])
} else {
enc.wf(floatAddDecimal(strconv.FormatFloat(f, 'f', -1, 32)))
}
enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 32)))
case reflect.Float64:
f := rv.Float()
if math.IsNaN(f) {
enc.wf("nan")
} else if math.IsInf(f, 0) {
enc.wf("%cinf", map[bool]byte{true: '-', false: '+'}[math.Signbit(f)])
} else {
enc.wf(floatAddDecimal(strconv.FormatFloat(f, 'f', -1, 64)))
}
enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 64)))
case reflect.Array, reflect.Slice:
enc.eArrayOrSliceElement(rv)
case reflect.Struct:
enc.eStruct(nil, rv, true)
case reflect.Map:
enc.eMap(nil, rv, true)
case reflect.Interface:
enc.eElement(rv.Elem())
case reflect.String:
enc.writeQuoted(rv.String())
default:
encPanic(fmt.Errorf("unexpected primitive type: %T", rv.Interface()))
panic(e("unexpected primitive type: %s", rv.Kind()))
}
}
// By the TOML spec, all floats must have a decimal with at least one number on
// either side.
// By the TOML spec, all floats must have a decimal with at least one
// number on either side.
func floatAddDecimal(fstr string) string {
if !strings.Contains(fstr, ".") {
return fstr + ".0"
@@ -285,14 +230,16 @@ func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) {
if isNil(trv) {
continue
}
panicIfInvalidKey(key)
enc.newline()
enc.wf("%s[[%s]]", enc.indentStr(key), key.maybeQuotedAll())
enc.newline()
enc.eMapOrStruct(key, trv, false)
enc.eMapOrStruct(key, trv)
}
}
func (enc *Encoder) eTable(key Key, rv reflect.Value) {
panicIfInvalidKey(key)
if len(key) == 1 {
// Output an extra newline between top-level tables.
// (The newline isn't written if nothing else has been written though.)
@@ -302,22 +249,21 @@ func (enc *Encoder) eTable(key Key, rv reflect.Value) {
enc.wf("%s[%s]", enc.indentStr(key), key.maybeQuotedAll())
enc.newline()
}
enc.eMapOrStruct(key, rv, false)
enc.eMapOrStruct(key, rv)
}
func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value, inline bool) {
func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value) {
switch rv := eindirect(rv); rv.Kind() {
case reflect.Map:
enc.eMap(key, rv, inline)
enc.eMap(key, rv)
case reflect.Struct:
enc.eStruct(key, rv, inline)
enc.eStruct(key, rv)
default:
// Should never happen?
panic("eTable: unhandled reflect.Value Kind: " + rv.Kind().String())
}
}
func (enc *Encoder) eMap(key Key, rv reflect.Value, inline bool) {
func (enc *Encoder) eMap(key Key, rv reflect.Value) {
rt := rv.Type()
if rt.Key().Kind() != reflect.String {
encPanic(errNonString)
@@ -335,76 +281,57 @@ func (enc *Encoder) eMap(key Key, rv reflect.Value, inline bool) {
}
}
var writeMapKeys = func(mapKeys []string, trailC bool) {
var writeMapKeys = func(mapKeys []string) {
sort.Strings(mapKeys)
for i, mapKey := range mapKeys {
val := rv.MapIndex(reflect.ValueOf(mapKey))
if isNil(val) {
for _, mapKey := range mapKeys {
mrv := rv.MapIndex(reflect.ValueOf(mapKey))
if isNil(mrv) {
// Don't write anything for nil fields.
continue
}
if inline {
enc.writeKeyValue(Key{mapKey}, val, true)
if trailC || i != len(mapKeys)-1 {
enc.wf(", ")
}
} else {
enc.encode(key.add(mapKey), val)
}
enc.encode(key.add(mapKey), mrv)
}
}
if inline {
enc.wf("{")
}
writeMapKeys(mapKeysDirect, len(mapKeysSub) > 0)
writeMapKeys(mapKeysSub, false)
if inline {
enc.wf("}")
}
writeMapKeys(mapKeysDirect)
writeMapKeys(mapKeysSub)
}
func (enc *Encoder) eStruct(key Key, rv reflect.Value, inline bool) {
func (enc *Encoder) eStruct(key Key, rv reflect.Value) {
// Write keys for fields directly under this key first, because if we write
// a field that creates a new table then all keys under it will be in that
// a field that creates a new table, then all keys under it will be in that
// table (not the one we're writing here).
//
// Fields is a [][]int: for fieldsDirect this always has one entry (the
// struct index). For fieldsSub it contains two entries: the parent field
// index from tv, and the field indexes for the fields of the sub.
var (
rt = rv.Type()
fieldsDirect, fieldsSub [][]int
addFields func(rt reflect.Type, rv reflect.Value, start []int)
)
rt := rv.Type()
var fieldsDirect, fieldsSub [][]int
var addFields func(rt reflect.Type, rv reflect.Value, start []int)
addFields = func(rt reflect.Type, rv reflect.Value, start []int) {
for i := 0; i < rt.NumField(); i++ {
f := rt.Field(i)
if f.PkgPath != "" && !f.Anonymous { /// Skip unexported fields.
// skip unexported fields
if f.PkgPath != "" && !f.Anonymous {
continue
}
frv := rv.Field(i)
// Treat anonymous struct fields with tag names as though they are
// not anonymous, like encoding/json does.
//
// Non-struct anonymous fields use the normal encoding logic.
if f.Anonymous {
t := f.Type
switch t.Kind() {
case reflect.Struct:
// Treat anonymous struct fields with
// tag names as though they are not
// anonymous, like encoding/json does.
if getOptions(f.Tag).name == "" {
addFields(t, frv, append(start, f.Index...))
addFields(t, frv, f.Index)
continue
}
case reflect.Ptr:
if t.Elem().Kind() == reflect.Struct && getOptions(f.Tag).name == "" {
if t.Elem().Kind() == reflect.Struct &&
getOptions(f.Tag).name == "" {
if !frv.IsNil() {
addFields(t.Elem(), frv.Elem(), append(start, f.Index...))
addFields(t.Elem(), frv.Elem(), f.Index)
}
continue
}
// Fall through to the normal field encoding logic below
// for non-struct anonymous fields.
}
}
@@ -417,49 +344,35 @@ func (enc *Encoder) eStruct(key Key, rv reflect.Value, inline bool) {
}
addFields(rt, rv, nil)
writeFields := func(fields [][]int) {
var writeFields = func(fields [][]int) {
for _, fieldIndex := range fields {
fieldType := rt.FieldByIndex(fieldIndex)
fieldVal := rv.FieldByIndex(fieldIndex)
if isNil(fieldVal) { /// Don't write anything for nil fields.
sft := rt.FieldByIndex(fieldIndex)
sf := rv.FieldByIndex(fieldIndex)
if isNil(sf) {
// Don't write anything for nil fields.
continue
}
opts := getOptions(fieldType.Tag)
opts := getOptions(sft.Tag)
if opts.skip {
continue
}
keyName := fieldType.Name
keyName := sft.Name
if opts.name != "" {
keyName = opts.name
}
if opts.omitempty && isEmpty(fieldVal) {
if opts.omitempty && isEmpty(sf) {
continue
}
if opts.omitzero && isZero(fieldVal) {
if opts.omitzero && isZero(sf) {
continue
}
if inline {
enc.writeKeyValue(Key{keyName}, fieldVal, true)
if fieldIndex[0] != len(fields)-1 {
enc.wf(", ")
}
} else {
enc.encode(key.add(keyName), fieldVal)
}
enc.encode(key.add(keyName), sf)
}
}
if inline {
enc.wf("{")
}
writeFields(fieldsDirect)
writeFields(fieldsSub)
if inline {
enc.wf("}")
}
}
// tomlTypeName returns the TOML type name of the Go value's type. It is
@@ -498,26 +411,13 @@ func tomlTypeOfGo(rv reflect.Value) tomlType {
switch rv.Interface().(type) {
case time.Time:
return tomlDatetime
case encoding.TextMarshaler:
case TextMarshaler:
return tomlString
default:
// Someone used a pointer receiver: we can make it work for pointer
// values.
if rv.CanAddr() {
_, ok := rv.Addr().Interface().(encoding.TextMarshaler)
if ok {
return tomlString
}
}
return tomlHash
}
default:
_, ok := rv.Interface().(encoding.TextMarshaler)
if ok {
return tomlString
}
encPanic(errors.New("unsupported type: " + rv.Kind().String()))
panic("") // Need *some* return value
panic("unexpected reflect.Kind: " + rv.Kind().String())
}
}
@@ -530,19 +430,30 @@ func tomlArrayType(rv reflect.Value) tomlType {
if isNil(rv) || !rv.IsValid() || rv.Len() == 0 {
return nil
}
/// Don't allow nil.
rvlen := rv.Len()
for i := 1; i < rvlen; i++ {
if tomlTypeOfGo(rv.Index(i)) == nil {
encPanic(errArrayNilElement)
}
}
firstType := tomlTypeOfGo(rv.Index(0))
if firstType == nil {
encPanic(errArrayNilElement)
}
rvlen := rv.Len()
for i := 1; i < rvlen; i++ {
elem := rv.Index(i)
switch elemType := tomlTypeOfGo(elem); {
case elemType == nil:
encPanic(errArrayNilElement)
case !typeEqual(firstType, elemType):
encPanic(errArrayMixedElementTypes)
}
}
// If we have a nested array, then we must make sure that the nested
// array contains ONLY primitives.
// This checks arbitrarily nested arrays.
if typeEqual(firstType, tomlArray) || typeEqual(firstType, tomlArrayHash) {
nest := tomlArrayType(eindirect(rv.Index(0)))
if typeEqual(nest, tomlHash) || typeEqual(nest, tomlArrayHash) {
encPanic(errArrayNoTable)
}
}
return firstType
}
@@ -600,20 +511,14 @@ func (enc *Encoder) newline() {
}
}
// Write a key/value pair:
//
// key = <any value>
//
// If inline is true it won't add a newline at the end.
func (enc *Encoder) writeKeyValue(key Key, val reflect.Value, inline bool) {
func (enc *Encoder) keyEqElement(key Key, val reflect.Value) {
if len(key) == 0 {
encPanic(errNoKey)
}
panicIfInvalidKey(key)
enc.wf("%s%s = ", enc.indentStr(key), key.maybeQuoted(len(key)-1))
enc.eElement(val)
if !inline {
enc.newline()
}
enc.newline()
}
func (enc *Encoder) wf(format string, v ...interface{}) {
@@ -648,3 +553,16 @@ func isNil(rv reflect.Value) bool {
return false
}
}
func panicIfInvalidKey(key Key) {
for _, k := range key {
if len(k) == 0 {
encPanic(e("Key '%s' is not a valid table name. Key names "+
"cannot be empty.", key.maybeQuotedAll()))
}
}
}
func isValidKeyName(s string) bool {
return len(s) != 0
}

19
vendor/github.com/BurntSushi/toml/encoding_types.go generated vendored Normal file
View File

@@ -0,0 +1,19 @@
// +build go1.2
package toml
// In order to support Go 1.1, we define our own TextMarshaler and
// TextUnmarshaler types. For Go 1.2+, we just alias them with the
// standard library interfaces.
import (
"encoding"
)
// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here
// so that Go 1.1 can be supported.
type TextMarshaler encoding.TextMarshaler
// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined
// here so that Go 1.1 can be supported.
type TextUnmarshaler encoding.TextUnmarshaler

View File

@@ -0,0 +1,18 @@
// +build !go1.2
package toml
// These interfaces were introduced in Go 1.2, so we add them manually when
// compiling for Go 1.1.
// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here
// so that Go 1.1 can be supported.
type TextMarshaler interface {
MarshalText() (text []byte, err error)
}
// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined
// here so that Go 1.1 can be supported.
type TextUnmarshaler interface {
UnmarshalText(text []byte) error
}

View File

@@ -1,3 +0,0 @@
module github.com/BurntSushi/toml
go 1.16

View File

View File

@@ -1,36 +0,0 @@
package internal
import "time"
// Timezones used for local datetime, date, and time TOML types.
//
// The exact way times and dates without a timezone should be interpreted is not
// well-defined in the TOML specification and left to the implementation. These
// defaults to current local timezone offset of the computer, but this can be
// changed by changing these variables before decoding.
//
// TODO:
// Ideally we'd like to offer people the ability to configure the used timezone
// by setting Decoder.Timezone and Encoder.Timezone; however, this is a bit
// tricky: the reason we use three different variables for this is to support
// round-tripping without these specific TZ names we wouldn't know which
// format to use.
//
// There isn't a good way to encode this right now though, and passing this sort
// of information also ties in to various related issues such as string format
// encoding, encoding of comments, etc.
//
// So, for the time being, just put this in internal until we can write a good
// comprehensive API for doing all of this.
//
// The reason they're exported is because they're referred from in e.g.
// internal/tag.
//
// Note that this behaviour is valid according to the TOML spec as the exact
// behaviour is left up to implementations.
var (
localOffset = func() int { _, o := time.Now().Zone(); return o }()
LocalDatetime = time.FixedZone("datetime-local", localOffset)
LocalDate = time.FixedZone("date-local", localOffset)
LocalTime = time.FixedZone("time-local", localOffset)
)

View File

@@ -2,8 +2,6 @@ package toml
import (
"fmt"
"reflect"
"runtime"
"strings"
"unicode"
"unicode/utf8"
@@ -31,7 +29,6 @@ const (
itemArrayTableStart
itemArrayTableEnd
itemKeyStart
itemKeyEnd
itemCommentStart
itemInlineTableStart
itemInlineTableEnd
@@ -67,9 +64,9 @@ type lexer struct {
state stateFn
items chan item
// Allow for backing up up to four runes.
// Allow for backing up up to three runes.
// This is necessary because TOML contains 3-rune tokens (""" and ''').
prevWidths [4]int
prevWidths [3]int
nprev int // how many of prevWidths are in use
// If we emit an eof, we can still back up, but it is not OK to call
// next again.
@@ -96,7 +93,6 @@ func (lx *lexer) nextItem() item {
return item
default:
lx.state = lx.state(lx)
//fmt.Printf(" STATE %-24s current: %-10q stack: %s\n", lx.state, lx.current(), lx.stack)
}
}
}
@@ -141,7 +137,7 @@ func (lx *lexer) emitTrim(typ itemType) {
func (lx *lexer) next() (r rune) {
if lx.atEOF {
panic("BUG in lexer: next called after EOF")
panic("next called after EOF")
}
if lx.pos >= len(lx.input) {
lx.atEOF = true
@@ -151,19 +147,12 @@ func (lx *lexer) next() (r rune) {
if lx.input[lx.pos] == '\n' {
lx.line++
}
lx.prevWidths[3] = lx.prevWidths[2]
lx.prevWidths[2] = lx.prevWidths[1]
lx.prevWidths[1] = lx.prevWidths[0]
if lx.nprev < 4 {
if lx.nprev < 3 {
lx.nprev++
}
r, w := utf8.DecodeRuneInString(lx.input[lx.pos:])
if r == utf8.RuneError {
lx.errorf("invalid UTF-8 byte at position %d (line %d): 0x%02x", lx.pos, lx.line, lx.input[lx.pos])
return utf8.RuneError
}
lx.prevWidths[0] = w
lx.pos += w
return r
@@ -174,19 +163,18 @@ func (lx *lexer) ignore() {
lx.start = lx.pos
}
// backup steps back one rune. Can be called 4 times between calls to next.
// backup steps back one rune. Can be called only twice between calls to next.
func (lx *lexer) backup() {
if lx.atEOF {
lx.atEOF = false
return
}
if lx.nprev < 1 {
panic("BUG in lexer: backed up too far")
panic("backed up too far")
}
w := lx.prevWidths[0]
lx.prevWidths[0] = lx.prevWidths[1]
lx.prevWidths[1] = lx.prevWidths[2]
lx.prevWidths[2] = lx.prevWidths[3]
lx.nprev--
lx.pos -= w
if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' {
@@ -281,9 +269,8 @@ func lexTopEnd(lx *lexer) stateFn {
lx.emit(itemEOF)
return nil
}
return lx.errorf(
"expected a top-level item to end with a newline, comment, or EOF, but got %q instead",
r)
return lx.errorf("expected a top-level item to end with a newline, "+
"comment, or EOF, but got %q instead", r)
}
// lexTable lexes the beginning of a table. Namely, it makes sure that
@@ -310,9 +297,8 @@ func lexTableEnd(lx *lexer) stateFn {
func lexArrayTableEnd(lx *lexer) stateFn {
if r := lx.next(); r != arrayTableEnd {
return lx.errorf(
"expected end of table array name delimiter %q, but got %q instead",
arrayTableEnd, r)
return lx.errorf("expected end of table array name delimiter %q, "+
"but got %q instead", arrayTableEnd, r)
}
lx.emit(itemArrayTableEnd)
return lexTopEnd
@@ -322,19 +308,32 @@ func lexTableNameStart(lx *lexer) stateFn {
lx.skip(isWhitespace)
switch r := lx.peek(); {
case r == tableEnd || r == eof:
return lx.errorf("unexpected end of table name (table names cannot be empty)")
return lx.errorf("unexpected end of table name " +
"(table names cannot be empty)")
case r == tableSep:
return lx.errorf("unexpected table separator (table names cannot be empty)")
return lx.errorf("unexpected table separator " +
"(table names cannot be empty)")
case r == stringStart || r == rawStringStart:
lx.ignore()
lx.push(lexTableNameEnd)
return lexQuotedName
return lexValue // reuse string lexing
default:
lx.push(lexTableNameEnd)
return lexBareName
return lexBareTableName
}
}
// lexBareTableName lexes the name of a table. It assumes that at least one
// valid character for the table has already been read.
func lexBareTableName(lx *lexer) stateFn {
r := lx.next()
if isBareKeyChar(r) {
return lexBareTableName
}
lx.backup()
lx.emit(itemText)
return lexTableNameEnd
}
// lexTableNameEnd reads the end of a piece of a table name, optionally
// consuming whitespace.
func lexTableNameEnd(lx *lexer) stateFn {
@@ -348,101 +347,63 @@ func lexTableNameEnd(lx *lexer) stateFn {
case r == tableEnd:
return lx.pop()
default:
return lx.errorf("expected '.' or ']' to end table name, but got %q instead", r)
return lx.errorf("expected '.' or ']' to end table name, "+
"but got %q instead", r)
}
}
// lexBareName lexes one part of a key or table.
//
// It assumes that at least one valid character for the table has already been
// read.
//
// Lexes only one part, e.g. only 'a' inside 'a.b'.
func lexBareName(lx *lexer) stateFn {
r := lx.next()
if isBareKeyChar(r) {
return lexBareName
}
lx.backup()
lx.emit(itemText)
return lx.pop()
}
// lexBareName lexes one part of a key or table.
//
// It assumes that at least one valid character for the table has already been
// read.
//
// Lexes only one part, e.g. only '"a"' inside '"a".b'.
func lexQuotedName(lx *lexer) stateFn {
r := lx.next()
switch {
case isWhitespace(r):
return lexSkip(lx, lexValue)
case r == stringStart:
lx.ignore() // ignore the '"'
return lexString
case r == rawStringStart:
lx.ignore() // ignore the "'"
return lexRawString
case r == eof:
return lx.errorf("unexpected EOF; expected value")
default:
return lx.errorf("expected value but found %q instead", r)
}
}
// lexKeyStart consumes all key parts until a '='.
// lexKeyStart consumes a key name up until the first non-whitespace character.
// lexKeyStart will ignore whitespace.
func lexKeyStart(lx *lexer) stateFn {
lx.skip(isWhitespace)
switch r := lx.peek(); {
case r == '=' || r == eof:
return lx.errorf("unexpected '=': key name appears blank")
case r == '.':
return lx.errorf("unexpected '.': keys cannot start with a '.'")
r := lx.peek()
switch {
case r == keySep:
return lx.errorf("unexpected key separator %q", keySep)
case isWhitespace(r) || isNL(r):
lx.next()
return lexSkip(lx, lexKeyStart)
case r == stringStart || r == rawStringStart:
lx.ignore()
fallthrough
default: // Bare key
lx.emit(itemKeyStart)
return lexKeyNameStart
lx.push(lexKeyEnd)
return lexValue // reuse string lexing
default:
lx.ignore()
lx.emit(itemKeyStart)
return lexBareKey
}
}
func lexKeyNameStart(lx *lexer) stateFn {
lx.skip(isWhitespace)
switch r := lx.peek(); {
case r == '=' || r == eof:
return lx.errorf("unexpected '='")
case r == '.':
return lx.errorf("unexpected '.'")
case r == stringStart || r == rawStringStart:
lx.ignore()
lx.push(lexKeyEnd)
return lexQuotedName
// lexBareKey consumes the text of a bare key. Assumes that the first character
// (which is not whitespace) has not yet been consumed.
func lexBareKey(lx *lexer) stateFn {
switch r := lx.next(); {
case isBareKeyChar(r):
return lexBareKey
case isWhitespace(r):
lx.backup()
lx.emit(itemText)
return lexKeyEnd
case r == keySep:
lx.backup()
lx.emit(itemText)
return lexKeyEnd
default:
lx.push(lexKeyEnd)
return lexBareName
return lx.errorf("bare keys cannot contain %q", r)
}
}
// lexKeyEnd consumes the end of a key and trims whitespace (up to the key
// separator).
func lexKeyEnd(lx *lexer) stateFn {
lx.skip(isWhitespace)
switch r := lx.next(); {
case r == keySep:
return lexSkip(lx, lexValue)
case isWhitespace(r):
return lexSkip(lx, lexKeyEnd)
case r == eof:
return lx.errorf("unexpected EOF; expected key separator %q", keySep)
case r == '.':
lx.ignore()
return lexKeyNameStart
case r == '=':
lx.emit(itemKeyEnd)
return lexSkip(lx, lexValue)
default:
return lx.errorf("expected '.' or '=', but got %q instead", r)
return lx.errorf("expected key separator %q, but got %q instead",
keySep, r)
}
}
@@ -489,15 +450,10 @@ func lexValue(lx *lexer) stateFn {
}
lx.ignore() // ignore the "'"
return lexRawString
case '+', '-':
return lexNumberStart
case '.': // special error case, be kind to users
return lx.errorf("floats must start with a digit, not '.'")
case 'i', 'n':
if (lx.accept('n') && lx.accept('f')) || (lx.accept('a') && lx.accept('n')) {
lx.emit(itemFloat)
return lx.pop()
}
case '-', '+':
return lexDecimalNumberStart
}
if unicode.IsLetter(r) {
// Be permissive here; lexBool will give a nice error if the
@@ -507,9 +463,6 @@ func lexValue(lx *lexer) stateFn {
lx.backup()
return lexBool
}
if r == eof {
return lx.errorf("unexpected EOF; expected value")
}
return lx.errorf("expected value but found %q instead", r)
}
@@ -554,8 +507,9 @@ func lexArrayValueEnd(lx *lexer) stateFn {
return lexArrayEnd
}
return lx.errorf(
"expected a comma or array terminator %q, but got %s instead",
arrayEnd, runeOrEOF(r))
"expected a comma or array terminator %q, but got %q instead",
arrayEnd, r,
)
}
// lexArrayEnd finishes the lexing of an array.
@@ -592,7 +546,8 @@ func lexInlineTableValue(lx *lexer) stateFn {
// key/value pair and the next pair (or the end of the table):
// it ignores whitespace and expects either a ',' or a '}'.
func lexInlineTableValueEnd(lx *lexer) stateFn {
switch r := lx.next(); {
r := lx.next()
switch {
case isWhitespace(r):
return lexSkip(lx, lexInlineTableValueEnd)
case isNL(r):
@@ -602,25 +557,12 @@ func lexInlineTableValueEnd(lx *lexer) stateFn {
return lexCommentStart
case r == comma:
lx.ignore()
lx.skip(isWhitespace)
if lx.peek() == '}' {
return lx.errorf("trailing comma not allowed in inline tables")
}
return lexInlineTableValue
case r == inlineTableEnd:
return lexInlineTableEnd
default:
return lx.errorf(
"expected a comma or an inline table terminator %q, but got %s instead",
inlineTableEnd, runeOrEOF(r))
}
}
func runeOrEOF(r rune) string {
if r == eof {
return "end of file"
}
return "'" + string(r) + "'"
return lx.errorf("expected a comma or an inline table terminator %q, "+
"but got %q instead", inlineTableEnd, r)
}
// lexInlineTableEnd finishes the lexing of an inline table.
@@ -637,9 +579,7 @@ func lexString(lx *lexer) stateFn {
r := lx.next()
switch {
case r == eof:
return lx.errorf(`unexpected EOF; expected '"'`)
case isControl(r) || r == '\r':
return lx.errorf("control characters are not allowed inside strings: '0x%02x'", r)
return lx.errorf("unexpected EOF")
case isNL(r):
return lx.errorf("strings cannot contain newlines")
case r == '\\':
@@ -658,40 +598,19 @@ func lexString(lx *lexer) stateFn {
// lexMultilineString consumes the inner contents of a string. It assumes that
// the beginning '"""' has already been consumed and ignored.
func lexMultilineString(lx *lexer) stateFn {
r := lx.next()
switch r {
switch lx.next() {
case eof:
return lx.errorf(`unexpected EOF; expected '"""'`)
case '\r':
if lx.peek() != '\n' {
return lx.errorf("control characters are not allowed inside strings: '0x%02x'", r)
}
return lexMultilineString
return lx.errorf("unexpected EOF")
case '\\':
return lexMultilineStringEscape
case stringEnd:
/// Found " → try to read two more "".
if lx.accept(stringEnd) {
if lx.accept(stringEnd) {
/// Peek ahead: the string can contain " and "", including at the
/// end: """str"""""
/// 6 or more at the end, however, is an error.
if lx.peek() == stringEnd {
/// Check if we already lexed 5 's; if so we have 6 now, and
/// that's just too many man!
if strings.HasSuffix(lx.current(), `"""""`) {
return lx.errorf(`unexpected '""""""'`)
}
lx.backup()
lx.backup()
return lexMultilineString
}
lx.backup() /// backup: don't include the """ in the item.
lx.backup()
lx.backup()
lx.backup()
lx.emit(itemMultilineString)
lx.next() /// Read over ''' again and discard it.
lx.next()
lx.next()
lx.next()
lx.ignore()
@@ -700,10 +619,6 @@ func lexMultilineString(lx *lexer) stateFn {
lx.backup()
}
}
if isControl(r) {
return lx.errorf("control characters are not allowed inside strings: '0x%02x'", r)
}
return lexMultilineString
}
@@ -713,9 +628,7 @@ func lexRawString(lx *lexer) stateFn {
r := lx.next()
switch {
case r == eof:
return lx.errorf(`unexpected EOF; expected "'"`)
case isControl(r) || r == '\r':
return lx.errorf("control characters are not allowed inside strings: '0x%02x'", r)
return lx.errorf("unexpected EOF")
case isNL(r):
return lx.errorf("strings cannot contain newlines")
case r == rawStringEnd:
@@ -732,38 +645,17 @@ func lexRawString(lx *lexer) stateFn {
// a string. It assumes that the beginning "'''" has already been consumed and
// ignored.
func lexMultilineRawString(lx *lexer) stateFn {
r := lx.next()
switch r {
switch lx.next() {
case eof:
return lx.errorf(`unexpected EOF; expected "'''"`)
case '\r':
if lx.peek() != '\n' {
return lx.errorf("control characters are not allowed inside strings: '0x%02x'", r)
}
return lexMultilineRawString
return lx.errorf("unexpected EOF")
case rawStringEnd:
/// Found ' → try to read two more ''.
if lx.accept(rawStringEnd) {
if lx.accept(rawStringEnd) {
/// Peek ahead: the string can contain ' and '', including at the
/// end: '''str'''''
/// 6 or more at the end, however, is an error.
if lx.peek() == rawStringEnd {
/// Check if we already lexed 5 's; if so we have 6 now, and
/// that's just too many man!
if strings.HasSuffix(lx.current(), "'''''") {
return lx.errorf(`unexpected "''''''"`)
}
lx.backup()
lx.backup()
return lexMultilineRawString
}
lx.backup() /// backup: don't include the ''' in the item.
lx.backup()
lx.backup()
lx.backup()
lx.emit(itemRawMultilineString)
lx.next() /// Read over ''' again and discard it.
lx.next()
lx.next()
lx.next()
lx.ignore()
@@ -772,10 +664,6 @@ func lexMultilineRawString(lx *lexer) stateFn {
lx.backup()
}
}
if isControl(r) {
return lx.errorf("control characters are not allowed inside strings: '0x%02x'", r)
}
return lexMultilineRawString
}
@@ -806,10 +694,6 @@ func lexStringEscape(lx *lexer) stateFn {
fallthrough
case '"':
fallthrough
case ' ', '\t':
// Inside """ .. """ strings you can use \ to escape newlines, and any
// amount of whitespace can be between the \ and \n.
fallthrough
case '\\':
return lx.pop()
case 'u':
@@ -817,7 +701,8 @@ func lexStringEscape(lx *lexer) stateFn {
case 'U':
return lexLongUnicodeEscape
}
return lx.errorf("invalid escape character %q; only the following escape characters are allowed: "+
return lx.errorf("invalid escape character %q; only the following "+
"escape characters are allowed: "+
`\b, \t, \n, \f, \r, \", \\, \uXXXX, and \UXXXXXXXX`, r)
}
@@ -826,9 +711,8 @@ func lexShortUnicodeEscape(lx *lexer) stateFn {
for i := 0; i < 4; i++ {
r = lx.next()
if !isHexadecimal(r) {
return lx.errorf(
`expected four hexadecimal digits after '\u', but got %q instead`,
lx.current())
return lx.errorf(`expected four hexadecimal digits after '\u', `+
"but got %q instead", lx.current())
}
}
return lx.pop()
@@ -839,33 +723,28 @@ func lexLongUnicodeEscape(lx *lexer) stateFn {
for i := 0; i < 8; i++ {
r = lx.next()
if !isHexadecimal(r) {
return lx.errorf(
`expected eight hexadecimal digits after '\U', but got %q instead`,
lx.current())
return lx.errorf(`expected eight hexadecimal digits after '\U', `+
"but got %q instead", lx.current())
}
}
return lx.pop()
}
// lexNumberOrDateStart processes the first character of a value which begins
// with a digit. It exists to catch values starting with '0', so that
// lexBaseNumberOrDate can differentiate base prefixed integers from other
// types.
// lexNumberOrDateStart consumes either an integer, a float, or datetime.
func lexNumberOrDateStart(lx *lexer) stateFn {
r := lx.next()
if isDigit(r) {
return lexNumberOrDate
}
switch r {
case '0':
return lexBaseNumberOrDate
case '_':
return lexNumber
case 'e', 'E':
return lexFloat
case '.':
return lx.errorf("floats must start with a digit, not '.'")
}
if !isDigit(r) {
// The only way to reach this state is if the value starts
// with a digit, so specifically treat anything else as an
// error.
return lx.errorf("expected a digit but got %q", r)
}
return lexNumberOrDate
return lx.errorf("expected a digit but got %q", r)
}
// lexNumberOrDate consumes either an integer, float or datetime.
@@ -875,10 +754,10 @@ func lexNumberOrDate(lx *lexer) stateFn {
return lexNumberOrDate
}
switch r {
case '-', ':':
case '-':
return lexDatetime
case '_':
return lexDecimalNumber
return lexNumber
case '.', 'e', 'E':
return lexFloat
}
@@ -896,156 +775,41 @@ func lexDatetime(lx *lexer) stateFn {
return lexDatetime
}
switch r {
case '-', ':', 'T', 't', ' ', '.', 'Z', 'z', '+':
case '-', 'T', ':', '.', 'Z', '+':
return lexDatetime
}
lx.backup()
lx.emitTrim(itemDatetime)
lx.emit(itemDatetime)
return lx.pop()
}
// lexHexInteger consumes a hexadecimal integer after seeing the '0x' prefix.
func lexHexInteger(lx *lexer) stateFn {
// lexNumberStart consumes either an integer or a float. It assumes that a sign
// has already been read, but that *no* digits have been consumed.
// lexNumberStart will move to the appropriate integer or float states.
func lexNumberStart(lx *lexer) stateFn {
// We MUST see a digit. Even floats have to start with a digit.
r := lx.next()
if isHexadecimal(r) {
return lexHexInteger
if !isDigit(r) {
if r == '.' {
return lx.errorf("floats must start with a digit, not '.'")
}
return lx.errorf("expected a digit but got %q", r)
}
switch r {
case '_':
return lexHexInteger
}
lx.backup()
lx.emit(itemInteger)
return lx.pop()
return lexNumber
}
// lexOctalInteger consumes an octal integer after seeing the '0o' prefix.
func lexOctalInteger(lx *lexer) stateFn {
r := lx.next()
if isOctal(r) {
return lexOctalInteger
}
switch r {
case '_':
return lexOctalInteger
}
lx.backup()
lx.emit(itemInteger)
return lx.pop()
}
// lexBinaryInteger consumes a binary integer after seeing the '0b' prefix.
func lexBinaryInteger(lx *lexer) stateFn {
r := lx.next()
if isBinary(r) {
return lexBinaryInteger
}
switch r {
case '_':
return lexBinaryInteger
}
lx.backup()
lx.emit(itemInteger)
return lx.pop()
}
// lexDecimalNumber consumes a decimal float or integer.
func lexDecimalNumber(lx *lexer) stateFn {
// lexNumber consumes an integer or a float after seeing the first digit.
func lexNumber(lx *lexer) stateFn {
r := lx.next()
if isDigit(r) {
return lexDecimalNumber
return lexNumber
}
switch r {
case '_':
return lexNumber
case '.', 'e', 'E':
return lexFloat
case '_':
return lexDecimalNumber
}
lx.backup()
lx.emit(itemInteger)
return lx.pop()
}
// lexDecimalNumber consumes the first digit of a number beginning with a sign.
// It assumes the sign has already been consumed. Values which start with a sign
// are only allowed to be decimal integers or floats.
//
// The special "nan" and "inf" values are also recognized.
func lexDecimalNumberStart(lx *lexer) stateFn {
r := lx.next()
// Special error cases to give users better error messages
switch r {
case 'i':
if !lx.accept('n') || !lx.accept('f') {
return lx.errorf("invalid float: '%s'", lx.current())
}
lx.emit(itemFloat)
return lx.pop()
case 'n':
if !lx.accept('a') || !lx.accept('n') {
return lx.errorf("invalid float: '%s'", lx.current())
}
lx.emit(itemFloat)
return lx.pop()
case '0':
p := lx.peek()
switch p {
case 'b', 'o', 'x':
return lx.errorf("cannot use sign with non-decimal numbers: '%s%c'", lx.current(), p)
}
case '.':
return lx.errorf("floats must start with a digit, not '.'")
}
if isDigit(r) {
return lexDecimalNumber
}
return lx.errorf("expected a digit but got %q", r)
}
// lexBaseNumberOrDate differentiates between the possible values which
// start with '0'. It assumes that before reaching this state, the initial '0'
// has been consumed.
func lexBaseNumberOrDate(lx *lexer) stateFn {
r := lx.next()
// Note: All datetimes start with at least two digits, so we don't
// handle date characters (':', '-', etc.) here.
if isDigit(r) {
return lexNumberOrDate
}
switch r {
case '_':
// Can only be decimal, because there can't be an underscore
// between the '0' and the base designator, and dates can't
// contain underscores.
return lexDecimalNumber
case '.', 'e', 'E':
return lexFloat
case 'b':
r = lx.peek()
if !isBinary(r) {
lx.errorf("not a binary number: '%s%c'", lx.current(), r)
}
return lexBinaryInteger
case 'o':
r = lx.peek()
if !isOctal(r) {
lx.errorf("not an octal number: '%s%c'", lx.current(), r)
}
return lexOctalInteger
case 'x':
r = lx.peek()
if !isHexadecimal(r) {
lx.errorf("not a hexidecimal number: '%s%c'", lx.current(), r)
}
return lexHexInteger
}
lx.backup()
@@ -1103,22 +867,21 @@ func lexCommentStart(lx *lexer) stateFn {
// It will consume *up to* the first newline character, and pass control
// back to the last state on the stack.
func lexComment(lx *lexer) stateFn {
switch r := lx.next(); {
case isNL(r) || r == eof:
lx.backup()
r := lx.peek()
if isNL(r) || r == eof {
lx.emit(itemText)
return lx.pop()
case isControl(r):
return lx.errorf("control characters are not allowed inside comments: '0x%02x'", r)
default:
return lexComment
}
lx.next()
return lexComment
}
// lexSkip ignores all slurped input and moves on to the next state.
func lexSkip(lx *lexer, nextState stateFn) stateFn {
lx.ignore()
return nextState
return func(lx *lexer) stateFn {
lx.ignore()
return nextState
}
}
// isWhitespace returns true if `r` is a whitespace character according
@@ -1131,16 +894,6 @@ func isNL(r rune) bool {
return r == '\n' || r == '\r'
}
// Control characters except \n, \t
func isControl(r rune) bool {
switch r {
case '\t', '\r', '\n':
return false
default:
return (r >= 0x00 && r <= 0x1f) || r == 0x7f
}
}
func isDigit(r rune) bool {
return r >= '0' && r <= '9'
}
@@ -1151,14 +904,6 @@ func isHexadecimal(r rune) bool {
(r >= 'A' && r <= 'F')
}
func isOctal(r rune) bool {
return r >= '0' && r <= '7'
}
func isBinary(r rune) bool {
return r == '0' || r == '1'
}
func isBareKeyChar(r rune) bool {
return (r >= 'A' && r <= 'Z') ||
(r >= 'a' && r <= 'z') ||
@@ -1167,17 +912,6 @@ func isBareKeyChar(r rune) bool {
r == '-'
}
func (s stateFn) String() string {
name := runtime.FuncForPC(reflect.ValueOf(s).Pointer()).Name()
if i := strings.LastIndexByte(name, '.'); i > -1 {
name = name[i+1:]
}
if s == nil {
name = "<nil>"
}
return name + "()"
}
func (itype itemType) String() string {
switch itype {
case itemError:
@@ -1204,18 +938,12 @@ func (itype itemType) String() string {
return "TableEnd"
case itemKeyStart:
return "KeyStart"
case itemKeyEnd:
return "KeyEnd"
case itemArray:
return "Array"
case itemArrayEnd:
return "ArrayEnd"
case itemCommentStart:
return "CommentStart"
case itemInlineTableStart:
return "InlineTableStart"
case itemInlineTableEnd:
return "InlineTableEnd"
}
panic(fmt.Sprintf("BUG: Unknown type '%d'.", int(itype)))
}

View File

@@ -1,14 +1,12 @@
package toml
import (
"errors"
"fmt"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/BurntSushi/toml/internal"
)
type parser struct {
@@ -16,54 +14,39 @@ type parser struct {
types map[string]tomlType
lx *lexer
ordered []Key // List of keys in the order that they appear in the TOML data.
context Key // Full key for the current hash in scope.
currentKey string // Base key name for everything except hashes.
approxLine int // Rough approximation of line number
implicits map[string]bool // Record implied keys (e.g. 'key.group.names').
// A list of keys in the order that they appear in the TOML data.
ordered []Key
// the full key for the current hash in scope
context Key
// the base key name for everything except hashes
currentKey string
// rough approximation of line number
approxLine int
// A map of 'key.group.names' to whether they were created implicitly.
implicits map[string]bool
}
// ParseError is used when a file can't be parsed: for example invalid integer
// literals, duplicate keys, etc.
type ParseError struct {
Message string
Line int
LastKey string
}
type parseError string
func (pe ParseError) Error() string {
return fmt.Sprintf("Near line %d (last key parsed '%s'): %s",
pe.Line, pe.LastKey, pe.Message)
func (pe parseError) Error() string {
return string(pe)
}
func parse(data string) (p *parser, err error) {
defer func() {
if r := recover(); r != nil {
var ok bool
if err, ok = r.(ParseError); ok {
if err, ok = r.(parseError); ok {
return
}
panic(r)
}
}()
// Read over BOM; do this here as the lexer calls utf8.DecodeRuneInString()
// which mangles stuff.
if strings.HasPrefix(data, "\xff\xfe") || strings.HasPrefix(data, "\xfe\xff") {
data = data[2:]
}
// Examine first few bytes for NULL bytes; this probably means it's a UTF-16
// file (second byte in surrogate pair being NULL). Again, do this here to
// avoid having to deal with UTF-8/16 stuff in the lexer.
ex := 6
if len(data) < 6 {
ex = len(data)
}
if strings.ContainsRune(data[:ex], 0) {
return nil, errors.New("files cannot contain NULL bytes; probably using UTF-16; TOML files must be UTF-8")
}
p = &parser{
mapping: make(map[string]interface{}),
types: make(map[string]tomlType),
@@ -83,17 +66,13 @@ func parse(data string) (p *parser, err error) {
}
func (p *parser) panicf(format string, v ...interface{}) {
msg := fmt.Sprintf(format, v...)
panic(ParseError{
Message: msg,
Line: p.approxLine,
LastKey: p.current(),
})
msg := fmt.Sprintf("Near line %d (last key parsed '%s'): %s",
p.approxLine, p.current(), fmt.Sprintf(format, v...))
panic(parseError(msg))
}
func (p *parser) next() item {
it := p.lx.nextItem()
//fmt.Printf("ITEM %-18s line %-3d │ %q\n", it.typ, it.line, it.val)
if it.typ == itemError {
p.panicf("%s", it.val)
}
@@ -118,63 +97,44 @@ func (p *parser) assertEqual(expected, got itemType) {
func (p *parser) topLevel(item item) {
switch item.typ {
case itemCommentStart: // # ..
case itemCommentStart:
p.approxLine = item.line
p.expect(itemText)
case itemTableStart: // [ .. ]
name := p.next()
p.approxLine = name.line
case itemTableStart:
kg := p.next()
p.approxLine = kg.line
var key Key
for ; name.typ != itemTableEnd && name.typ != itemEOF; name = p.next() {
key = append(key, p.keyString(name))
for ; kg.typ != itemTableEnd && kg.typ != itemEOF; kg = p.next() {
key = append(key, p.keyString(kg))
}
p.assertEqual(itemTableEnd, name.typ)
p.assertEqual(itemTableEnd, kg.typ)
p.addContext(key, false)
p.establishContext(key, false)
p.setType("", tomlHash)
p.ordered = append(p.ordered, key)
case itemArrayTableStart: // [[ .. ]]
name := p.next()
p.approxLine = name.line
case itemArrayTableStart:
kg := p.next()
p.approxLine = kg.line
var key Key
for ; name.typ != itemArrayTableEnd && name.typ != itemEOF; name = p.next() {
key = append(key, p.keyString(name))
for ; kg.typ != itemArrayTableEnd && kg.typ != itemEOF; kg = p.next() {
key = append(key, p.keyString(kg))
}
p.assertEqual(itemArrayTableEnd, name.typ)
p.assertEqual(itemArrayTableEnd, kg.typ)
p.addContext(key, true)
p.establishContext(key, true)
p.setType("", tomlArrayHash)
p.ordered = append(p.ordered, key)
case itemKeyStart: // key = ..
outerContext := p.context
/// Read all the key parts (e.g. 'a' and 'b' in 'a.b')
k := p.next()
p.approxLine = k.line
var key Key
for ; k.typ != itemKeyEnd && k.typ != itemEOF; k = p.next() {
key = append(key, p.keyString(k))
}
p.assertEqual(itemKeyEnd, k.typ)
case itemKeyStart:
kname := p.next()
p.approxLine = kname.line
p.currentKey = p.keyString(kname)
/// The current key is the last part.
p.currentKey = key[len(key)-1]
/// All the other parts (if any) are the context; need to set each part
/// as implicit.
context := key[:len(key)-1]
for i := range context {
p.addImplicitContext(append(p.context, context[i:i+1]...))
}
/// Set value.
val, typ := p.value(p.next(), false)
p.set(p.currentKey, val, typ)
val, typ := p.value(p.next())
p.setValue(p.currentKey, val)
p.setType(p.currentKey, typ)
p.ordered = append(p.ordered, p.context.add(p.currentKey))
/// Remove the context we added (preserving any context from [tbl] lines).
p.context = outerContext
p.currentKey = ""
default:
p.bug("Unexpected type at top level: %s", item.typ)
@@ -188,253 +148,180 @@ func (p *parser) keyString(it item) string {
return it.val
case itemString, itemMultilineString,
itemRawString, itemRawMultilineString:
s, _ := p.value(it, false)
s, _ := p.value(it)
return s.(string)
default:
p.bug("Unexpected key type: %s", it.typ)
panic("unreachable")
}
panic("unreachable")
}
var datetimeRepl = strings.NewReplacer(
"z", "Z",
"t", "T",
" ", "T")
// value translates an expected value from the lexer into a Go value wrapped
// as an empty interface.
func (p *parser) value(it item, parentIsArray bool) (interface{}, tomlType) {
func (p *parser) value(it item) (interface{}, tomlType) {
switch it.typ {
case itemString:
return p.replaceEscapes(it.val), p.typeOfPrimitive(it)
case itemMultilineString:
return p.replaceEscapes(stripFirstNewline(stripEscapedNewlines(it.val))), p.typeOfPrimitive(it)
trimmed := stripFirstNewline(stripEscapedWhitespace(it.val))
return p.replaceEscapes(trimmed), p.typeOfPrimitive(it)
case itemRawString:
return it.val, p.typeOfPrimitive(it)
case itemRawMultilineString:
return stripFirstNewline(it.val), p.typeOfPrimitive(it)
case itemInteger:
return p.valueInteger(it)
case itemFloat:
return p.valueFloat(it)
case itemBool:
switch it.val {
case "true":
return true, p.typeOfPrimitive(it)
case "false":
return false, p.typeOfPrimitive(it)
default:
p.bug("Expected boolean value, but got '%s'.", it.val)
}
p.bug("Expected boolean value, but got '%s'.", it.val)
case itemInteger:
if !numUnderscoresOK(it.val) {
p.panicf("Invalid integer %q: underscores must be surrounded by digits",
it.val)
}
val := strings.Replace(it.val, "_", "", -1)
num, err := strconv.ParseInt(val, 10, 64)
if err != nil {
// Distinguish integer values. Normally, it'd be a bug if the lexer
// provides an invalid integer, but it's possible that the number is
// out of range of valid values (which the lexer cannot determine).
// So mark the former as a bug but the latter as a legitimate user
// error.
if e, ok := err.(*strconv.NumError); ok &&
e.Err == strconv.ErrRange {
p.panicf("Integer '%s' is out of the range of 64-bit "+
"signed integers.", it.val)
} else {
p.bug("Expected integer value, but got '%s'.", it.val)
}
}
return num, p.typeOfPrimitive(it)
case itemFloat:
parts := strings.FieldsFunc(it.val, func(r rune) bool {
switch r {
case '.', 'e', 'E':
return true
}
return false
})
for _, part := range parts {
if !numUnderscoresOK(part) {
p.panicf("Invalid float %q: underscores must be "+
"surrounded by digits", it.val)
}
}
if !numPeriodsOK(it.val) {
// As a special case, numbers like '123.' or '1.e2',
// which are valid as far as Go/strconv are concerned,
// must be rejected because TOML says that a fractional
// part consists of '.' followed by 1+ digits.
p.panicf("Invalid float %q: '.' must be followed "+
"by one or more digits", it.val)
}
val := strings.Replace(it.val, "_", "", -1)
num, err := strconv.ParseFloat(val, 64)
if err != nil {
if e, ok := err.(*strconv.NumError); ok &&
e.Err == strconv.ErrRange {
p.panicf("Float '%s' is out of the range of 64-bit "+
"IEEE-754 floating-point numbers.", it.val)
} else {
p.panicf("Invalid float value: %q", it.val)
}
}
return num, p.typeOfPrimitive(it)
case itemDatetime:
return p.valueDatetime(it)
var t time.Time
var ok bool
var err error
for _, format := range []string{
"2006-01-02T15:04:05Z07:00",
"2006-01-02T15:04:05",
"2006-01-02",
} {
t, err = time.ParseInLocation(format, it.val, time.Local)
if err == nil {
ok = true
break
}
}
if !ok {
p.panicf("Invalid TOML Datetime: %q.", it.val)
}
return t, p.typeOfPrimitive(it)
case itemArray:
return p.valueArray(it)
array := make([]interface{}, 0)
types := make([]tomlType, 0)
for it = p.next(); it.typ != itemArrayEnd; it = p.next() {
if it.typ == itemCommentStart {
p.expect(itemText)
continue
}
val, typ := p.value(it)
array = append(array, val)
types = append(types, typ)
}
return array, p.typeOfArray(types)
case itemInlineTableStart:
return p.valueInlineTable(it, parentIsArray)
default:
p.bug("Unexpected value type: %s", it.typ)
var (
hash = make(map[string]interface{})
outerContext = p.context
outerKey = p.currentKey
)
p.context = append(p.context, p.currentKey)
p.currentKey = ""
for it := p.next(); it.typ != itemInlineTableEnd; it = p.next() {
if it.typ != itemKeyStart {
p.bug("Expected key start but instead found %q, around line %d",
it.val, p.approxLine)
}
if it.typ == itemCommentStart {
p.expect(itemText)
continue
}
// retrieve key
k := p.next()
p.approxLine = k.line
kname := p.keyString(k)
// retrieve value
p.currentKey = kname
val, typ := p.value(p.next())
// make sure we keep metadata up to date
p.setType(kname, typ)
p.ordered = append(p.ordered, p.context.add(p.currentKey))
hash[kname] = val
}
p.context = outerContext
p.currentKey = outerKey
return hash, tomlHash
}
p.bug("Unexpected value type: %s", it.typ)
panic("unreachable")
}
func (p *parser) valueInteger(it item) (interface{}, tomlType) {
if !numUnderscoresOK(it.val) {
p.panicf("Invalid integer %q: underscores must be surrounded by digits", it.val)
}
if numHasLeadingZero(it.val) {
p.panicf("Invalid integer %q: cannot have leading zeroes", it.val)
}
num, err := strconv.ParseInt(it.val, 0, 64)
if err != nil {
// Distinguish integer values. Normally, it'd be a bug if the lexer
// provides an invalid integer, but it's possible that the number is
// out of range of valid values (which the lexer cannot determine).
// So mark the former as a bug but the latter as a legitimate user
// error.
if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange {
p.panicf("Integer '%s' is out of the range of 64-bit signed integers.", it.val)
} else {
p.bug("Expected integer value, but got '%s'.", it.val)
}
}
return num, p.typeOfPrimitive(it)
}
func (p *parser) valueFloat(it item) (interface{}, tomlType) {
parts := strings.FieldsFunc(it.val, func(r rune) bool {
switch r {
case '.', 'e', 'E':
return true
}
return false
})
for _, part := range parts {
if !numUnderscoresOK(part) {
p.panicf("Invalid float %q: underscores must be surrounded by digits", it.val)
}
}
if len(parts) > 0 && numHasLeadingZero(parts[0]) {
p.panicf("Invalid float %q: cannot have leading zeroes", it.val)
}
if !numPeriodsOK(it.val) {
// As a special case, numbers like '123.' or '1.e2',
// which are valid as far as Go/strconv are concerned,
// must be rejected because TOML says that a fractional
// part consists of '.' followed by 1+ digits.
p.panicf("Invalid float %q: '.' must be followed by one or more digits", it.val)
}
val := strings.Replace(it.val, "_", "", -1)
if val == "+nan" || val == "-nan" { // Go doesn't support this, but TOML spec does.
val = "nan"
}
num, err := strconv.ParseFloat(val, 64)
if err != nil {
if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange {
p.panicf("Float '%s' is out of the range of 64-bit IEEE-754 floating-point numbers.", it.val)
} else {
p.panicf("Invalid float value: %q", it.val)
}
}
return num, p.typeOfPrimitive(it)
}
var dtTypes = []struct {
fmt string
zone *time.Location
}{
{time.RFC3339Nano, time.Local},
{"2006-01-02T15:04:05.999999999", internal.LocalDatetime},
{"2006-01-02", internal.LocalDate},
{"15:04:05.999999999", internal.LocalTime},
}
func (p *parser) valueDatetime(it item) (interface{}, tomlType) {
it.val = datetimeRepl.Replace(it.val)
var (
t time.Time
ok bool
err error
)
for _, dt := range dtTypes {
t, err = time.ParseInLocation(dt.fmt, it.val, dt.zone)
if err == nil {
ok = true
break
}
}
if !ok {
p.panicf("Invalid TOML Datetime: %q.", it.val)
}
return t, p.typeOfPrimitive(it)
}
func (p *parser) valueArray(it item) (interface{}, tomlType) {
p.setType(p.currentKey, tomlArray)
// p.setType(p.currentKey, typ)
var (
array []interface{}
types []tomlType
)
for it = p.next(); it.typ != itemArrayEnd; it = p.next() {
if it.typ == itemCommentStart {
p.expect(itemText)
continue
}
val, typ := p.value(it, true)
array = append(array, val)
types = append(types, typ)
}
return array, tomlArray
}
func (p *parser) valueInlineTable(it item, parentIsArray bool) (interface{}, tomlType) {
var (
hash = make(map[string]interface{})
outerContext = p.context
outerKey = p.currentKey
)
p.context = append(p.context, p.currentKey)
prevContext := p.context
p.currentKey = ""
p.addImplicit(p.context)
p.addContext(p.context, parentIsArray)
/// Loop over all table key/value pairs.
for it := p.next(); it.typ != itemInlineTableEnd; it = p.next() {
if it.typ == itemCommentStart {
p.expect(itemText)
continue
}
/// Read all key parts.
k := p.next()
p.approxLine = k.line
var key Key
for ; k.typ != itemKeyEnd && k.typ != itemEOF; k = p.next() {
key = append(key, p.keyString(k))
}
p.assertEqual(itemKeyEnd, k.typ)
/// The current key is the last part.
p.currentKey = key[len(key)-1]
/// All the other parts (if any) are the context; need to set each part
/// as implicit.
context := key[:len(key)-1]
for i := range context {
p.addImplicitContext(append(p.context, context[i:i+1]...))
}
/// Set the value.
val, typ := p.value(p.next(), false)
p.set(p.currentKey, val, typ)
p.ordered = append(p.ordered, p.context.add(p.currentKey))
hash[p.currentKey] = val
/// Restore context.
p.context = prevContext
}
p.context = outerContext
p.currentKey = outerKey
return hash, tomlHash
}
// numHasLeadingZero checks if this number has leading zeroes, allowing for '0',
// +/- signs, and base prefixes.
func numHasLeadingZero(s string) bool {
if len(s) > 1 && s[0] == '0' && isDigit(rune(s[1])) { // >1 to allow "0" and isDigit to allow 0x
return true
}
if len(s) > 2 && (s[0] == '-' || s[0] == '+') && s[1] == '0' {
return true
}
return false
}
// numUnderscoresOK checks whether each underscore in s is surrounded by
// characters that are not underscores.
func numUnderscoresOK(s string) bool {
switch s {
case "nan", "+nan", "-nan", "inf", "-inf", "+inf":
return true
}
accept := false
for _, r := range s {
if r == '_' {
if !accept {
return false
}
accept = false
continue
}
// isHexadecimal is a superset of all the permissable characters
// surrounding an underscore.
accept = isHexadecimal(r)
accept = true
}
return accept
}
@@ -451,12 +338,13 @@ func numPeriodsOK(s string) bool {
return !period
}
// Set the current context of the parser, where the context is either a hash or
// an array of hashes, depending on the value of the `array` parameter.
// establishContext sets the current context of the parser,
// where the context is either a hash or an array of hashes. Which one is
// set depends on the value of the `array` parameter.
//
// Establishing the context also makes sure that the key isn't a duplicate, and
// will create implicit hashes automatically.
func (p *parser) addContext(key Key, array bool) {
func (p *parser) establishContext(key Key, array bool) {
var ok bool
// Always start at the top level and drill down for our context.
@@ -495,7 +383,7 @@ func (p *parser) addContext(key Key, array bool) {
// list of tables for it.
k := key[len(key)-1]
if _, ok := hashContext[k]; !ok {
hashContext[k] = make([]map[string]interface{}, 0, 4)
hashContext[k] = make([]map[string]interface{}, 0, 5)
}
// Add a new table. But make sure the key hasn't already been used
@@ -503,7 +391,8 @@ func (p *parser) addContext(key Key, array bool) {
if hash, ok := hashContext[k].([]map[string]interface{}); ok {
hashContext[k] = append(hash, make(map[string]interface{}))
} else {
p.panicf("Key '%s' was already created and cannot be used as an array.", keyContext)
p.panicf("Key '%s' was already created and cannot be used as "+
"an array.", keyContext)
}
} else {
p.setValue(key[len(key)-1], make(map[string]interface{}))
@@ -511,22 +400,15 @@ func (p *parser) addContext(key Key, array bool) {
p.context = append(p.context, key[len(key)-1])
}
// set calls setValue and setType.
func (p *parser) set(key string, val interface{}, typ tomlType) {
p.setValue(p.currentKey, val)
p.setType(p.currentKey, typ)
}
// setValue sets the given key to the given value in the current context.
// It will make sure that the key hasn't already been defined, account for
// implicit key groups.
func (p *parser) setValue(key string, value interface{}) {
var (
tmpHash interface{}
ok bool
hash = p.mapping
keyContext Key
)
var tmpHash interface{}
var ok bool
hash := p.mapping
keyContext := make(Key, 0)
for _, k := range p.context {
keyContext = append(keyContext, k)
if tmpHash, ok = hash[k]; !ok {
@@ -540,26 +422,24 @@ func (p *parser) setValue(key string, value interface{}) {
case map[string]interface{}:
hash = t
default:
p.panicf("Key '%s' has already been defined.", keyContext)
p.bug("Expected hash to have type 'map[string]interface{}', but "+
"it has '%T' instead.", tmpHash)
}
}
keyContext = append(keyContext, key)
if _, ok := hash[key]; ok {
// Normally redefining keys isn't allowed, but the key could have been
// defined implicitly and it's allowed to be redefined concretely. (See
// the `valid/implicit-and-explicit-after.toml` in toml-test)
// Typically, if the given key has already been set, then we have
// to raise an error since duplicate keys are disallowed. However,
// it's possible that a key was previously defined implicitly. In this
// case, it is allowed to be redefined concretely. (See the
// `tests/valid/implicit-and-explicit-after.toml` test in `toml-test`.)
//
// But we have to make sure to stop marking it as an implicit. (So that
// another redefinition provokes an error.)
//
// Note that since it has already been defined (as a hash), we don't
// want to overwrite it. So our business is done.
if p.isArray(keyContext) {
p.removeImplicit(keyContext)
hash[key] = value
return
}
if p.isImplicit(keyContext) {
p.removeImplicit(keyContext)
return
@@ -569,7 +449,6 @@ func (p *parser) setValue(key string, value interface{}) {
// key, which is *always* wrong.
p.panicf("Key '%s' has already been defined.", keyContext)
}
hash[key] = value
}
@@ -589,15 +468,21 @@ func (p *parser) setType(key string, typ tomlType) {
p.types[keyContext.String()] = typ
}
// Implicit keys need to be created when tables are implied in "a.b.c.d = 1" and
// "[a.b.c]" (the "a", "b", and "c" hashes are never created explicitly).
func (p *parser) addImplicit(key Key) { p.implicits[key.String()] = true }
func (p *parser) removeImplicit(key Key) { p.implicits[key.String()] = false }
func (p *parser) isImplicit(key Key) bool { return p.implicits[key.String()] }
func (p *parser) isArray(key Key) bool { return p.types[key.String()] == tomlArray }
func (p *parser) addImplicitContext(key Key) {
p.addImplicit(key)
p.addContext(key, false)
// addImplicit sets the given Key as having been created implicitly.
func (p *parser) addImplicit(key Key) {
p.implicits[key.String()] = true
}
// removeImplicit stops tagging the given key as having been implicitly
// created.
func (p *parser) removeImplicit(key Key) {
p.implicits[key.String()] = false
}
// isImplicit returns true if the key group pointed to by the key was created
// implicitly.
func (p *parser) isImplicit(key Key) bool {
return p.implicits[key.String()]
}
// current returns the full key name of the current context.
@@ -612,54 +497,20 @@ func (p *parser) current() string {
}
func stripFirstNewline(s string) string {
if len(s) > 0 && s[0] == '\n' {
return s[1:]
}
if len(s) > 1 && s[0] == '\r' && s[1] == '\n' {
return s[2:]
}
return s
}
// Remove newlines inside triple-quoted strings if a line ends with "\".
func stripEscapedNewlines(s string) string {
split := strings.Split(s, "\n")
if len(split) < 1 {
if len(s) == 0 || s[0] != '\n' {
return s
}
return s[1:]
}
escNL := false // Keep track of the last non-blank line was escaped.
for i, line := range split {
line = strings.TrimRight(line, " \t\r")
if len(line) == 0 || line[len(line)-1] != '\\' {
split[i] = strings.TrimRight(split[i], "\r")
if !escNL && i != len(split)-1 {
split[i] += "\n"
}
continue
}
escBS := true
for j := len(line) - 1; j >= 0 && line[j] == '\\'; j-- {
escBS = !escBS
}
if escNL {
line = strings.TrimLeft(line, " \t\r")
}
escNL = !escBS
if escBS {
split[i] += "\n"
continue
}
split[i] = line[:len(line)-1] // Remove \
if len(split)-1 > i {
split[i+1] = strings.TrimLeft(split[i+1], " \t\r")
func stripEscapedWhitespace(s string) string {
esc := strings.Split(s, "\\\n")
if len(esc) > 1 {
for i := 1; i < len(esc); i++ {
esc[i] = strings.TrimLeftFunc(esc[i], unicode.IsSpace)
}
}
return strings.Join(split, "")
return strings.Join(esc, "")
}
func (p *parser) replaceEscapes(str string) string {
@@ -682,9 +533,6 @@ func (p *parser) replaceEscapes(str string) string {
default:
p.bug("Expected valid escape code after \\, but got %q.", s[r])
return ""
case ' ', '\t':
p.panicf("invalid escape: '\\%c'", s[r])
return ""
case 'b':
replaced = append(replaced, rune(0x0008))
r += 1
@@ -737,3 +585,8 @@ func (p *parser) asciiEscapeToUnicode(bs []byte) rune {
}
return rune(hex)
}
func isStringType(ty itemType) bool {
return ty == itemString || ty == itemMultilineString ||
ty == itemRawString || ty == itemRawMultilineString
}

1
vendor/github.com/BurntSushi/toml/session.vim generated vendored Normal file
View File

@@ -0,0 +1 @@
au BufWritePost *.go silent!make tags > /dev/null 2>&1

View File

@@ -68,3 +68,24 @@ func (p *parser) typeOfPrimitive(lexItem item) tomlType {
p.bug("Cannot infer primitive type of lex item '%s'.", lexItem)
panic("unreachable")
}
// typeOfArray returns a tomlType for an array given a list of types of its
// values.
//
// In the current spec, if an array is homogeneous, then its type is always
// "Array". If the array is not homogeneous, an error is generated.
func (p *parser) typeOfArray(types []tomlType) tomlType {
// Empty arrays are cool.
if len(types) == 0 {
return tomlArray
}
theType := types[0]
for _, t := range types[1:] {
if !typeEqual(theType, t) {
p.panicf("Array contains values of type '%s' and '%s', but "+
"arrays must be homogeneous.", theType, t)
}
}
return tomlArray
}

View File

@@ -1 +0,0 @@
* @microsoft/containerplat

Some files were not shown because too many files have changed in this diff Show More