├── README.md ├── tests └── swupd-wrapper │ ├── content │ └── usr │ │ ├── share │ │ └── -dash-file │ │ └── bin │ │ ├── foo.sh │ │ └── test.sh │ ├── content2 │ └── usr │ │ └── bin │ │ ├── bar.sh │ │ └── test.sh │ ├── example-config.toml │ └── test-runner.sh ├── .gitignore ├── data ├── 3rd-party-update.service └── 3rd-party-update.timer ├── go.mod ├── go.sum ├── post-job ├── main.go ├── operations │ └── process.go └── cmd │ └── root.go ├── swupd-wrapper ├── main.go ├── cmd │ ├── list.go │ ├── update.go │ ├── add.go │ ├── remove.go │ └── root.go └── operations │ ├── remove.go │ ├── list.go │ ├── update.go │ └── add.go ├── docs ├── 3rd-party-post.1.rst ├── swupd-3rd-party.1.rst └── mixer-user-bundler.1.rst ├── Makefile ├── cublib ├── config.go ├── postjob.go └── utils.go ├── 3rd-party-post.1 ├── swupd-3rd-party.1 ├── mixer-user-bundler.1 ├── COPYING └── clr-user-bundles.py /README.md: -------------------------------------------------------------------------------- 1 | # Deprecated 2 | -------------------------------------------------------------------------------- /tests/swupd-wrapper/content/usr/share/-dash-file: -------------------------------------------------------------------------------- 1 | evil-dash 2 | -------------------------------------------------------------------------------- /tests/swupd-wrapper/content/usr/bin/foo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo fooenv 4 | -------------------------------------------------------------------------------- /tests/swupd-wrapper/content2/usr/bin/bar.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | echo barenv 4 | -------------------------------------------------------------------------------- /tests/swupd-wrapper/content/usr/bin/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | echo baz 4 | foo.sh 5 | -------------------------------------------------------------------------------- /tests/swupd-wrapper/content2/usr/bin/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | echo zab 4 | bar.sh 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.xz 2 | privatekey.pem 3 | Swupd_Root.pem 4 | 3rd-party-post 5 | swupd-3rd-party 6 | vendor/ 7 | -------------------------------------------------------------------------------- /data/3rd-party-update.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Update 3rd-Party Software Content 3 | 4 | [Service] 5 | Type=oneshot 6 | ExecStart=/usr/bin/swupd-3rd-party update 7 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/clearlinux/clr-user-bundles 2 | 3 | require ( 4 | github.com/BurntSushi/toml v0.3.1 5 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 6 | github.com/spf13/cobra v0.0.3 7 | github.com/spf13/pflag v1.0.3 // indirect 8 | ) 9 | -------------------------------------------------------------------------------- /tests/swupd-wrapper/example-config.toml: -------------------------------------------------------------------------------- 1 | [upstream] 2 | url = "https://download.clearlinux.org/update" 3 | version = @VERSION@ 4 | 5 | [bundle] 6 | name = "test" 7 | description = "test user bundle" 8 | includes = [ "os-core", "os-core-update" ] 9 | url = "@TESTDIR@" 10 | bin = [ "/usr/bin/test.sh" ] 11 | -------------------------------------------------------------------------------- /data/3rd-party-update.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Automatically Install 3rd-Party Software Updates 3 | Documentation=man:swupd-3rd-party_update 4 | 5 | [Timer] 6 | OnBootSec=64min 7 | OnUnitActiveSec=64min 8 | AccuracySec=300 9 | RandomizedDelaySec=2700 10 | 11 | [Install] 12 | WantedBy=multi-user.target 13 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 4 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 5 | github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= 6 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 7 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 8 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 9 | -------------------------------------------------------------------------------- /post-job/main.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/clearlinux/clr-user-bundles/post-job/cmd" 19 | ) 20 | 21 | func main() { 22 | cmd.Execute() 23 | } 24 | -------------------------------------------------------------------------------- /swupd-wrapper/main.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/clearlinux/clr-user-bundles/swupd-wrapper/cmd" 19 | ) 20 | 21 | func main() { 22 | cmd.Execute() 23 | } 24 | -------------------------------------------------------------------------------- /swupd-wrapper/cmd/list.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "github.com/spf13/cobra" 19 | "github.com/clearlinux/clr-user-bundles/swupd-wrapper/operations" 20 | ) 21 | 22 | var listCmd = &cobra.Command{ 23 | Use: "list", 24 | Short: "list 3rd party bundle metadata", 25 | Run: func(cmd *cobra.Command, args []string) { 26 | operations.List(StateDirectory, ContentDirectory) 27 | }, 28 | } 29 | 30 | func init() { 31 | rootCmd.AddCommand(listCmd) 32 | } 33 | -------------------------------------------------------------------------------- /post-job/operations/process.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package operations 16 | 17 | import ( 18 | "log" 19 | "github.com/clearlinux/clr-user-bundles/cublib" 20 | ) 21 | 22 | func ProcessContent(statedir string, contentdir string) { 23 | // GetLock causes program exit on failure to acquire lockfile 24 | cublib.GetLock(statedir) 25 | defer cublib.ReleaseLock(statedir) 26 | err := cublib.PostProcess(statedir, contentdir) 27 | if err != nil { 28 | log.Fatalf("%s", err) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /swupd-wrapper/cmd/update.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "github.com/spf13/cobra" 19 | "github.com/clearlinux/clr-user-bundles/swupd-wrapper/operations" 20 | ) 21 | 22 | var updateCmd = &cobra.Command{ 23 | Use: "update", 24 | Short: "Update 3rd party bundle content", 25 | Args: func(cmd *cobra.Command, args []string) error { 26 | if cmd.PersistentFlags().Changed("skip-post") { 27 | skipPost = true 28 | } 29 | 30 | return nil 31 | }, 32 | Run: func(cmd *cobra.Command, args []string) { 33 | operations.Update(StateDirectory, ContentDirectory, skipPost) 34 | }, 35 | } 36 | 37 | func init() { 38 | updateCmd.PersistentFlags().BoolVarP(&skipPost, "skip-post", "p", false, "Skip running post-3rd-party hooks") 39 | rootCmd.AddCommand(updateCmd) 40 | } 41 | -------------------------------------------------------------------------------- /swupd-wrapper/cmd/add.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "github.com/spf13/cobra" 20 | "github.com/clearlinux/clr-user-bundles/swupd-wrapper/operations" 21 | ) 22 | 23 | var addCmd = &cobra.Command{ 24 | Use: "add [URI to 3rd party content]", 25 | Short: "Add 3rd party bundle content", 26 | Args: func(cmd *cobra.Command, args []string) error { 27 | if len(args) != 1 { 28 | return fmt.Errorf("Invalid URI specification") 29 | } 30 | if cmd.PersistentFlags().Changed("skip-post") { 31 | skipPost = true 32 | } 33 | return nil 34 | }, 35 | Run: func(cmd *cobra.Command, args []string) { 36 | operations.Add(args[0], StateDirectory, ContentDirectory, skipPost) 37 | }, 38 | } 39 | 40 | func init() { 41 | addCmd.PersistentFlags().BoolVarP(&skipPost, "skip-post", "p", false, "Skip running post-3rd-party hooks") 42 | rootCmd.AddCommand(addCmd) 43 | } 44 | -------------------------------------------------------------------------------- /docs/3rd-party-post.1.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | 3rd-party-post 3 | ============== 4 | 5 | ------------------------------ 6 | Post job for 3rd-party content 7 | ------------------------------ 8 | 9 | :Copyright: \(C) 2019 Intel Corporation, CC-BY-SA-3.0 10 | :Manual section: 1 11 | 12 | 13 | SYNOPSIS 14 | ======== 15 | 16 | ``3rd-party-post `` 17 | 18 | 19 | DESCRIPTION 20 | =========== 21 | 22 | ``3rd-party-post``\(1) is a tool for creating, updating and cleaning up 23 | 3rd-party content artifacts that were added through ``swupd-3rd-party``\(1). 24 | 25 | Contents installed (by default) under /opt/3rd-party are processed and 26 | configured applications will have runner scripts generated for them under 27 | /opt/3rd-party/bin (which should be added to the PATH as the last entry). 28 | 29 | 30 | OPTIONS 31 | ======= 32 | 33 | The following options are applicable to be used to modify the core behavior and 34 | resources that ``3rd-party-post`` uses. 35 | 36 | - ``-h, --help`` 37 | 38 | Display general help information. 39 | 40 | - ``-c, --contentdir`` 41 | 42 | Changes the installation directory for 3rd-party content. 43 | 44 | - ``-s, --statedir`` 45 | 46 | Changes the statedir used by ``swupd``\(1). 47 | 48 | 49 | EXIT STATUS 50 | =========== 51 | 52 | On success, 0 is returned. A non-zero return code indicates a failure. 53 | 54 | SEE ALSO 55 | -------- 56 | 57 | * ``swupd``\(1) 58 | * ``swupd-3rd-party``\(1) 59 | * https://github.com/clearlinux/swupd-client 60 | * https://clearlinux.org/documentation/ 61 | -------------------------------------------------------------------------------- /swupd-wrapper/cmd/remove.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "github.com/spf13/cobra" 20 | "github.com/clearlinux/clr-user-bundles/swupd-wrapper/operations" 21 | ) 22 | 23 | var removeCmd = &cobra.Command{ 24 | Use: "remove [URI to 3rd party content] [BUNDLE-NAME]", 25 | Short: "Remove 3rd party bundle content", 26 | Args: func(cmd *cobra.Command, args []string) error { 27 | if len(args) != 2 { 28 | return fmt.Errorf("Invalid arguments") 29 | } 30 | if cmd.PersistentFlags().Changed("skip-post") { 31 | skipPost = true 32 | } 33 | 34 | return nil 35 | }, 36 | Run: func(cmd *cobra.Command, args []string) { 37 | operations.Remove(StateDirectory, ContentDirectory, args[0], args[1], skipPost, true) 38 | }, 39 | } 40 | 41 | func init() { 42 | removeCmd.PersistentFlags().BoolVarP(&skipPost, "skip-post", "p", false, "Skip running post-3rd-party hooks") 43 | rootCmd.AddCommand(removeCmd) 44 | } 45 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test clean all 2 | 3 | MANPAGES := \ 4 | 3rd-party-post.1 \ 5 | mixer-user-bundler.1 \ 6 | swupd-3rd-party.1 7 | 8 | all: vendor man 9 | (cd ./swupd-wrapper && go build -mod=vendor -o ../swupd-3rd-party) 10 | (cd ./post-job && go build -mod=vendor -o ../3rd-party-post) 11 | 12 | install: all 13 | install -D -m 00755 swupd-3rd-party $(DESTDIR)/usr/bin/swupd-3rd-party 14 | install -D -m 00755 3rd-party-post $(DESTDIR)/usr/bin/3rd-party-post 15 | install -D -m 00755 clr-user-bundles.py $(DESTDIR)/usr/bin/mixer-user-bundler 16 | install -D -m 00644 data/3rd-party-update.service $(DESTDIR)/usr/lib/systemd/system/3rd-party-update.service 17 | install -D -m 00644 data/3rd-party-update.timer $(DESTDIR)/usr/lib/systemd/system/3rd-party-update.timer 18 | install -D -m 00644 3rd-party-post.1 $(DESTDIR)/usr/share/man/man1/3rd-party-post.1 19 | install -D -m 00644 mixer-user-bundler.1 $(DESTDIR)/usr/share/man/man1/mixer-user-bundler.1 20 | install -D -m 00644 swupd-3rd-party.1 $(DESTDIR)/usr/share/man/man1/swupd-3rd-party.1 21 | 22 | clean: 23 | rm -f swupd-3rd-party 3rd-party-post clr-user-bundles-*.tar.xz 24 | rm -fr vendor 25 | 26 | vendor: go.mod 27 | go mod vendor 28 | 29 | test: all 30 | tests/swupd-wrapper/test-runner.sh tests/swupd-wrapper 31 | 32 | man: $(MANPAGES) 33 | 34 | %: docs/%.rst 35 | rst2man.py "$<" > "$@.tmp" && mv -f "$@.tmp" "$@" 36 | 37 | dist: vendor 38 | $(eval TMP := $(shell mktemp -d)) 39 | cp -r . $(TMP)/clr-user-bundles-$(VERSION) 40 | (cd $(TMP)/clr-user-bundles-$(VERSION); git reset --hard $(VERSION); git clean -xf; rm -fr .git .gitignore) 41 | tar -C $(TMP) -cf clr-user-bundles-$(VERSION).tar . 42 | xz clr-user-bundles-$(VERSION).tar 43 | rm -fr $(TMP) 44 | -------------------------------------------------------------------------------- /swupd-wrapper/cmd/root.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "github.com/spf13/cobra" 21 | ) 22 | 23 | var rootCmd = &cobra.Command{ 24 | Use: "3rd-party", 25 | Long: `swupd 3rd-party bundle manager.`, 26 | Args: func(cmd *cobra.Command, args []string) error { 27 | if StateDirectory[0] != '/' { 28 | return fmt.Errorf("statedir path must be absolute") 29 | } 30 | if ContentDirectory[0] != '/' { 31 | return fmt.Errorf("contentdir path must be absolute") 32 | } 33 | return nil 34 | }, 35 | Run: func(cmd *cobra.Command, args []string) { 36 | cmd.Print(cmd.UsageString()) 37 | }, 38 | } 39 | 40 | var StateDirectory string 41 | var ContentDirectory string 42 | var skipPost bool 43 | 44 | func Execute() { 45 | if err := rootCmd.Execute(); err != nil { 46 | fmt.Println(err) 47 | os.Exit(1) 48 | } 49 | } 50 | 51 | func init() { 52 | rootCmd.PersistentFlags().StringVarP(&StateDirectory, "statedir", "s", "/var/lib/swupd", "swupd state directory") 53 | rootCmd.PersistentFlags().StringVarP(&ContentDirectory, "contentdir", "c", "/opt/3rd-party", "3rd-party content directory") 54 | } 55 | -------------------------------------------------------------------------------- /post-job/cmd/root.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "log" 20 | "github.com/spf13/cobra" 21 | "github.com/clearlinux/clr-user-bundles/post-job/operations" 22 | ) 23 | 24 | var rootCmd = &cobra.Command{ 25 | Long: `Process installed swupd 3rd party content into a usable form.`, 26 | Args: func(cmd *cobra.Command, args []string) error { 27 | if StateDirectory[0] != '/' { 28 | return fmt.Errorf("statedir path must be absolute") 29 | } 30 | if ContentDirectory[0] != '/' { 31 | return fmt.Errorf("contentdir path must be absolute") 32 | } 33 | return nil 34 | }, 35 | Run: func(cmd *cobra.Command, args []string) { 36 | operations.ProcessContent(StateDirectory, ContentDirectory) 37 | }, 38 | } 39 | 40 | var StateDirectory string 41 | var ContentDirectory string 42 | 43 | func Execute() { 44 | if err := rootCmd.Execute(); err != nil { 45 | log.Fatalln(err) 46 | } 47 | } 48 | 49 | func init() { 50 | rootCmd.PersistentFlags().StringVarP(&StateDirectory, "statedir", "s", "/var/lib/swupd", "swupd state directory") 51 | rootCmd.PersistentFlags().StringVarP(&ContentDirectory, "contentdir", "c", "/opt/3rd-party", "3rd-party content directory") 52 | } 53 | -------------------------------------------------------------------------------- /swupd-wrapper/operations/remove.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package operations 16 | 17 | import ( 18 | "log" 19 | "os" 20 | "path" 21 | "github.com/clearlinux/clr-user-bundles/cublib" 22 | ) 23 | 24 | func Remove(statedir string, contentdir string, uri string, name string, skipPost bool, lock bool) { 25 | if lock { 26 | // GetLock causes program exit on failure to acquire lockfile 27 | cublib.GetLock(statedir) 28 | defer cublib.ReleaseLock(statedir) 29 | } 30 | encodedName := cublib.GetEncodedBundleName(uri, name) 31 | pstatedir := path.Join(statedir, "3rd-party", encodedName) 32 | chrootdir := path.Join(contentdir, "chroot", encodedName) 33 | err := os.RemoveAll(pstatedir) 34 | if err != nil { 35 | log.Printf("WARNING: Unable to remove 3rd-party state directory (%s): %s", pstatedir, err) 36 | } 37 | err = os.RemoveAll(chrootdir) 38 | if err != nil { 39 | log.Printf("WARNING: Unable to remove 3rd-party content directory (%s): %s", chrootdir, err) 40 | } 41 | err = os.Remove(chrootdir + ".toml") 42 | if err != nil { 43 | log.Printf("WARNING: Unable to remove 3rd-party config (%s): %s", chrootdir + ".toml", err) 44 | } 45 | if skipPost { 46 | return 47 | } 48 | if err = cublib.PostProcess(statedir, contentdir); err != nil { 49 | log.Fatalf("%s", err) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /cublib/config.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cublib 16 | 17 | import ( 18 | "bytes" 19 | "io" 20 | "io/ioutil" 21 | "net/http" 22 | "net/url" 23 | "os" 24 | "github.com/BurntSushi/toml" 25 | ) 26 | 27 | type TomlConfig struct { 28 | Bundle BundleConfig 29 | } 30 | 31 | type BundleConfig struct { 32 | Name string 33 | Description string 34 | Includes []string 35 | URL string 36 | Bin []string 37 | } 38 | 39 | func ReadConfig(buffer io.Reader) (TomlConfig, error) { 40 | var config TomlConfig 41 | 42 | if _, err := toml.DecodeReader(buffer, &config); err != nil { 43 | return TomlConfig{}, err 44 | } 45 | 46 | return config, nil 47 | } 48 | 49 | func GetConfig(uri string) (TomlConfig, error) { 50 | url, err := url.ParseRequestURI(uri) 51 | if err != nil { 52 | return TomlConfig{}, err 53 | } 54 | var breader io.Reader 55 | if url.Scheme == "file" { 56 | buffer, err := ioutil.ReadFile(url.Path) 57 | if err != nil { 58 | return TomlConfig{}, err 59 | } 60 | breader = bytes.NewReader(buffer) 61 | } else { 62 | resp, err := http.Get(uri) 63 | if err != nil { 64 | return TomlConfig{}, err 65 | } else { 66 | defer resp.Body.Close() 67 | breader = resp.Body 68 | } 69 | } 70 | 71 | return ReadConfig(breader) 72 | } 73 | 74 | func WriteConfig(outPath string, config TomlConfig, overwrite bool) error { 75 | out, err := os.Create(outPath) 76 | if err != nil { 77 | return err 78 | } 79 | if err = toml.NewEncoder(out).Encode(config); err != nil { 80 | return err 81 | } 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /3rd-party-post.1: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH 3RD-PARTY-POST 1 "" "" "" 4 | .SH NAME 5 | 3rd-party-post \- Post job for 3rd-party content 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fB3rd\-party\-post \fP 36 | .SH DESCRIPTION 37 | .sp 38 | \fB3rd\-party\-post\fP(1) is a tool for creating, updating and cleaning up 39 | 3rd\-party content artifacts that were added through \fBswupd\-3rd\-party\fP(1). 40 | .sp 41 | Contents installed (by default) under /opt/3rd\-party are processed and 42 | configured applications will have runner scripts generated for them under 43 | /opt/3rd\-party/bin (which should be added to the PATH as the last entry). 44 | .SH OPTIONS 45 | .sp 46 | The following options are applicable to be used to modify the core behavior and 47 | resources that \fB3rd\-party\-post\fP uses. 48 | .INDENT 0.0 49 | .IP \(bu 2 50 | \fB\-h, \-\-help\fP 51 | .sp 52 | Display general help information. 53 | .IP \(bu 2 54 | \fB\-c, \-\-contentdir\fP 55 | .sp 56 | Changes the installation directory for 3rd\-party content. 57 | .IP \(bu 2 58 | \fB\-s, \-\-statedir\fP 59 | .sp 60 | Changes the statedir used by \fBswupd\fP(1). 61 | .UNINDENT 62 | .SH EXIT STATUS 63 | .sp 64 | On success, 0 is returned. A non\-zero return code indicates a failure. 65 | .SS SEE ALSO 66 | .INDENT 0.0 67 | .IP \(bu 2 68 | \fBswupd\fP(1) 69 | .IP \(bu 2 70 | \fBswupd\-3rd\-party\fP(1) 71 | .IP \(bu 2 72 | \fI\%https://github.com/clearlinux/swupd\-client\fP 73 | .IP \(bu 2 74 | \fI\%https://clearlinux.org/documentation/\fP 75 | .UNINDENT 76 | .SH COPYRIGHT 77 | (C) 2019 Intel Corporation, CC-BY-SA-3.0 78 | .\" Generated by docutils manpage writer. 79 | . 80 | -------------------------------------------------------------------------------- /docs/swupd-3rd-party.1.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | swupd-3rd-party 3 | =============== 4 | 5 | ----------------------------------- 6 | swupd wrapper for 3rd-party content 7 | ----------------------------------- 8 | 9 | :Copyright: \(C) 2019 Intel Corporation, CC-BY-SA-3.0 10 | :Manual section: 1 11 | 12 | 13 | SYNOPSIS 14 | ======== 15 | 16 | ``swupd-3rd-party [SUBCOMMANDS] `` 17 | 18 | 19 | DESCRIPTION 20 | =========== 21 | 22 | ``swupd-3rd-party``\(1) is a wrapper for ``swupd``\(1) that enables working with 23 | 3rd-party content in a way that conforms to the ``stateless``\(7) configuration 24 | and content repositories built by ``mixer-user-bundler``\(1). 25 | 26 | Contents are installed by default under /opt/3rd-party with hooks using 27 | ``3rd-party-post``\(1) for configured applications to be available under 28 | /opt/3rd-party/bin which should be added to the PATH as the last entry. 29 | 30 | 31 | OPTIONS 32 | ======= 33 | 34 | The following options are applicable to all subcommands, and can be 35 | used to modify the core behavior and resources that ``swupd-3rd-party`` 36 | uses. 37 | 38 | - ``-h, --help`` 39 | 40 | Display general help information. 41 | 42 | - ``-c, --contentdir`` 43 | 44 | Changes the installation directory for 3rd-party content. 45 | 46 | - ``-s, --statedir`` 47 | 48 | Changes the statedir used by ``swupd``. 49 | 50 | 51 | SUBCOMMANDS 52 | =========== 53 | 54 | ``add`` [URI] 55 | 56 | Add 3rd-party repo based on URI of the content. Content must be signed 57 | with a certificate trusted by the system trust store. 58 | 59 | addflags: 60 | 61 | - ``-p, --skip-post`` Skip running ``3rd-party-post`` processing. 62 | 63 | ``list`` 64 | 65 | Display installed 3rd-party content and its configured settings. 66 | 67 | ``remove`` [URI] [BUNDLE] 68 | 69 | Remove 3rd-party repo based on URI and BUNDLE name of the content. 70 | 71 | removeflags: 72 | 73 | - ``-p, --skip-post`` Skip running ``3rd-party-post`` processing. 74 | 75 | ``update`` 76 | 77 | Update all 3rd-party repositories on the system. 78 | 79 | updateflags: 80 | 81 | - ``-p, --skip-post`` Skip running ``3rd-party-post`` processing. 82 | 83 | 84 | EXIT STATUS 85 | =========== 86 | 87 | On success, 0 is returned. A non-zero return code indicates a failure. 88 | 89 | SEE ALSO 90 | -------- 91 | 92 | * ``3rd-party-post``\(1) 93 | * ``mixer-user-bundler``\(1) 94 | * ``swupd``\(1) 95 | * https://github.com/clearlinux/swupd-client 96 | * https://clearlinux.org/documentation/ 97 | -------------------------------------------------------------------------------- /swupd-wrapper/operations/list.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package operations 16 | 17 | import ( 18 | "fmt" 19 | "io/ioutil" 20 | "log" 21 | "path" 22 | "path/filepath" 23 | "github.com/clearlinux/clr-user-bundles/cublib" 24 | ) 25 | 26 | func List(statedir string, contentdir string) { 27 | // GetLock causes program exit on failure to acquire lockfile 28 | cublib.GetLock(statedir) 29 | defer cublib.ReleaseLock(statedir) 30 | chrootdir := path.Join(contentdir, "chroot") 31 | dlist, err := ioutil.ReadDir(chrootdir) 32 | if err != nil { 33 | log.Fatalf("Unable to read 3rd-party content directory (%s): %s", chrootdir, err) 34 | } 35 | 36 | fmt.Println("Installed 3rd-party bundles") 37 | for _, p := range dlist { 38 | // chroot dir should only be chroot directories and conf files so skip the conf files 39 | // as it is easier to make those names from the directory names 40 | if ext := filepath.Ext(p.Name()); ext != "" { 41 | continue 42 | } 43 | confPath := "file://" + path.Join(chrootdir, p.Name()) + ".toml" 44 | conf, err := cublib.GetConfig(confPath) 45 | if err != nil { 46 | log.Printf("WARNING: Unable to read 3rd-party config (%s): %s", confPath, err) 47 | continue 48 | } 49 | // Includes can be updated by the 3rd-party repo so show the updated config in that case 50 | newConfPath := "file://" + path.Join(chrootdir, p.Name(), "usr", "user-config.toml") 51 | newConf, err := cublib.GetConfig(newConfPath) 52 | if err != nil { 53 | log.Printf("WARNING: Unable to read updated 3rd-party config (%s): %s", newConfPath, err) 54 | continue 55 | } 56 | fmt.Println("") 57 | fmt.Println("Included Bundles:") 58 | fmt.Printf("Name: %-28s\n", conf.Bundle.Name) 59 | fmt.Printf("Description: %-28s\n", conf.Bundle.Description) 60 | fmt.Printf("URL: %-28s\n", conf.Bundle.URL) 61 | if len(conf.Bundle.Bin) > 0 { 62 | fmt.Println("Applications:") 63 | for _, app := range conf.Bundle.Bin { 64 | fmt.Printf(" %-28s\n", app) 65 | } 66 | } 67 | if len(newConf.Bundle.Includes) > 0 { 68 | fmt.Println("Included Bundles:") 69 | for _, include := range newConf.Bundle.Includes { 70 | fmt.Printf(" %-28s\n", include) 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /docs/mixer-user-bundler.1.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | mixer-user-bundler 3 | ================== 4 | 5 | ------------------------ 6 | 3rd-party bundle builder 7 | ------------------------ 8 | 9 | :Copyright: \(C) 2019 Intel Corporation, CC-BY-SA-3.0 10 | :Manual section: 1 11 | 12 | 13 | SYNOPSIS 14 | ======== 15 | 16 | ``mixer-user-bundler [STATEDIR] [CHROOTDIR] [CONFIG] `` 17 | 18 | 19 | DESCRIPTION 20 | =========== 21 | 22 | ``mixer-user-bundler``\(1) is a software update generator that takes a content 23 | root \(CHROOTDIR) and CONFIG to create update metadata consumable by ``swupd``\(1). 24 | 25 | ``mixer-user-bundler`` will modify chrootdir with additional content required 26 | for ``swupd`` so it is recommended to make a copy of chrootdir that is used 27 | only with ``mixer-user-bundler``. The STATEDIR content needs to be maintained 28 | between runs of ``mixer-user-bundler`` and the ``swupd`` update content is 29 | located within it at STATEDIR/www/update which needs to be available to clients 30 | to make use of this content. 31 | 32 | The output of ``mixer-user-bundler`` is a set of manifests readable by ``swupd`` 33 | as well as all the OS content ``swupd`` needs to perform its update operations. 34 | The OS content includes all the files in an update as well as zero and 35 | delta-packs for improved update performance. The content that 36 | ``mixer-user-bundler`` produces is tied to a specific format so that ``swupd`` 37 | is guaranteed to understand it if the client is using the right version of 38 | ``swupd``. The way users are expected to interact with the content is through 39 | ``swupd-3rd-party``\(1) and ``3rd-party-post``\(1) commands that setup the 40 | content on the end users system based on the configuration provided when 41 | creating the user bundle. See ``3rd-party-post``\(1), ``swupd-3rd-party``\(1), 42 | ``swupd``\(1) and ``os-format``\(7) for more details. 43 | 44 | 45 | OPTIONS 46 | ======= 47 | 48 | The following options are applicable to be used to modify the core behavior and 49 | resources that ``mixer-user-bundler`` uses. 50 | 51 | - ``-h, --help`` 52 | 53 | Display general help information. 54 | 55 | 56 | FILES 57 | ===== 58 | 59 | `CONFIG.toml` 60 | 61 | The mixer-user-bundler configuration file. This is a toml* formatted file 62 | containing two tables. The first is an upstream table with url and version 63 | keys, where url is a string that references the upstream content that 64 | the user bundle is based on and version is an integer of the upstream 65 | version the user bundle is to be built against. The second table is a 66 | bundle table with name, description, includes, url and bin keys. The name 67 | key is a string with the name of the user bundle being created, the 68 | description key is a short string describing what the bundle's purpose is 69 | for an end user, the includes key is an array of strings that are names 70 | of upstream bundles that are to be installed in order for the user bundle 71 | to function, the url key is a string which has the update url for the user 72 | bundle content (pointing to the STATEDIR/www/update content) and the bin 73 | key is an array of strings with absolute paths that resolve to executables 74 | in the CHROOTDIR that are to be exposed as executables to the end user. 75 | 76 | 77 | EXIT STATUS 78 | =========== 79 | 80 | On success, 0 is returned. A non-zero return code indicates a failure. 81 | 82 | SEE ALSO 83 | -------- 84 | 85 | * ``3rd-party-post``\(1) 86 | * ``swupd``\(1) 87 | * ``swupd-3rd-party``\(1) 88 | * ``os-format``\(7) 89 | * https://github.com/clearlinux/swupd-client 90 | * https://clearlinux.org/documentation/ 91 | * https://github.com/toml-lang/toml 92 | -------------------------------------------------------------------------------- /cublib/postjob.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cublib 16 | 17 | import ( 18 | "fmt" 19 | "io/ioutil" 20 | "log" 21 | "os" 22 | "path" 23 | "path/filepath" 24 | ) 25 | 26 | func setupBins(statedir string, contentdir string, installdir string, bins []string) error { 27 | scriptTemplate := `#!/bin/bash 28 | 29 | export PATH=%s:%s 30 | export LD_LIBRARY_PATH=%s:%s 31 | ` 32 | internalBinPath := fmt.Sprintf("%s/usr/bin", installdir) 33 | internalLdPath := fmt.Sprintf("%s/usr/lib64", installdir) 34 | targetPath := path.Join(contentdir, ".bin") 35 | err := os.MkdirAll(targetPath, 0755) 36 | if err != nil { 37 | return err 38 | } 39 | envPath := os.Getenv("PATH") 40 | envLdPath := os.Getenv("LD_LIBRARY_PATH") 41 | binScript := fmt.Sprintf(scriptTemplate, internalBinPath, envPath, internalLdPath, envLdPath) 42 | for _, b := range bins { 43 | if _, err = os.Lstat(path.Join(installdir, b)); err != nil { 44 | log.Printf("WARNING: Application %s set to be installed but not found in %s", b, installdir) 45 | continue 46 | } 47 | fullBinScript := fmt.Sprintf("%s%s \"$@\"\n", binScript, path.Join(installdir, b)) 48 | err = ioutil.WriteFile(path.Join(targetPath, path.Base(b)), []byte(fullBinScript), 0755) 49 | if err != nil { 50 | return err 51 | } 52 | } 53 | return nil 54 | } 55 | 56 | func stageContent(contentdir string) error { 57 | items := []string{"bin"} 58 | for _, item := range items { 59 | old := path.Join(contentdir, item) 60 | new := path.Join(contentdir, fmt.Sprintf(".%s", item)) 61 | err := os.RemoveAll(old) 62 | if err != nil && !os.IsNotExist(err) { 63 | return err 64 | } 65 | _, err = os.Stat(new) 66 | if os.IsNotExist(err) { 67 | continue 68 | } 69 | err = os.Rename(new, old) 70 | if err != nil { 71 | return err 72 | } 73 | } 74 | return nil 75 | } 76 | 77 | func PostProcess(statedir string, contentdir string) error { 78 | pstatedir := path.Join(statedir, "3rd-party") 79 | chrootdir := path.Join(contentdir, "chroot") 80 | dlist, err := ioutil.ReadDir(chrootdir) 81 | if err != nil { 82 | return fmt.Errorf("Unable to read 3rd-party content directory (%s): %s", pstatedir, err) 83 | } 84 | 85 | for _, p := range dlist { 86 | // chroot dir should only be chroot directories and conf files so skip the conf files 87 | // as it is easier to make those names from the directory names 88 | if ext := filepath.Ext(p.Name()); ext != "" { 89 | continue 90 | } 91 | confPath := "file://" + path.Join(chrootdir, p.Name()) + ".toml" 92 | conf, err := GetConfig(confPath) 93 | if err != nil { 94 | log.Printf("WARNING: Unable to read 3rd party config (%s): %s", confPath, err) 95 | continue 96 | } 97 | if err = setupBins(pstatedir, contentdir, path.Join(chrootdir, p.Name()), conf.Bundle.Bin); err != nil { 98 | log.Printf("WARNING: Unable to create bin scripts for %s: %s", conf.Bundle.Name, err) 99 | } 100 | } 101 | if err = stageContent(contentdir); err != nil { 102 | return fmt.Errorf("User content not staged successfully to %s: %s", contentdir, err) 103 | } 104 | 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /swupd-3rd-party.1: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH SWUPD-3RD-PARTY 1 "" "" "" 4 | .SH NAME 5 | swupd-3rd-party \- swupd wrapper for 3rd-party content 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fBswupd\-3rd\-party [SUBCOMMANDS] \fP 36 | .SH DESCRIPTION 37 | .sp 38 | \fBswupd\-3rd\-party\fP(1) is a wrapper for \fBswupd\fP(1) that enables working with 39 | 3rd\-party content in a way that conforms to the \fBstateless\fP(7) configuration 40 | and content repositories built by \fBmixer\-user\-bundler\fP(1). 41 | .sp 42 | Contents are installed by default under /opt/3rd\-party with hooks using 43 | \fB3rd\-party\-post\fP(1) for configured applications to be available under 44 | /opt/3rd\-party/bin which should be added to the PATH as the last entry. 45 | .SH OPTIONS 46 | .sp 47 | The following options are applicable to all subcommands, and can be 48 | used to modify the core behavior and resources that \fBswupd\-3rd\-party\fP 49 | uses. 50 | .INDENT 0.0 51 | .IP \(bu 2 52 | \fB\-h, \-\-help\fP 53 | .sp 54 | Display general help information. 55 | .IP \(bu 2 56 | \fB\-c, \-\-contentdir\fP 57 | .sp 58 | Changes the installation directory for 3rd\-party content. 59 | .IP \(bu 2 60 | \fB\-s, \-\-statedir\fP 61 | .sp 62 | Changes the statedir used by \fBswupd\fP\&. 63 | .UNINDENT 64 | .SH SUBCOMMANDS 65 | .sp 66 | \fBadd\fP [URI] 67 | .INDENT 0.0 68 | .INDENT 3.5 69 | Add 3rd\-party repo based on URI of the content. Content must be signed 70 | with a certificate trusted by the system trust store. 71 | .sp 72 | addflags: 73 | .INDENT 0.0 74 | .IP \(bu 2 75 | \fB\-p, \-\-skip\-post\fP Skip running \fB3rd\-party\-post\fP processing. 76 | .UNINDENT 77 | .UNINDENT 78 | .UNINDENT 79 | .sp 80 | \fBlist\fP 81 | .INDENT 0.0 82 | .INDENT 3.5 83 | Display installed 3rd\-party content and its configured settings. 84 | .UNINDENT 85 | .UNINDENT 86 | .sp 87 | \fBremove\fP [URI] [BUNDLE] 88 | .INDENT 0.0 89 | .INDENT 3.5 90 | Remove 3rd\-party repo based on URI and BUNDLE name of the content. 91 | .sp 92 | removeflags: 93 | .INDENT 0.0 94 | .IP \(bu 2 95 | \fB\-p, \-\-skip\-post\fP Skip running \fB3rd\-party\-post\fP processing. 96 | .UNINDENT 97 | .UNINDENT 98 | .UNINDENT 99 | .sp 100 | \fBupdate\fP 101 | .INDENT 0.0 102 | .INDENT 3.5 103 | Update all 3rd\-party repositories on the system. 104 | .sp 105 | updateflags: 106 | .INDENT 0.0 107 | .IP \(bu 2 108 | \fB\-p, \-\-skip\-post\fP Skip running \fB3rd\-party\-post\fP processing. 109 | .UNINDENT 110 | .UNINDENT 111 | .UNINDENT 112 | .SH EXIT STATUS 113 | .sp 114 | On success, 0 is returned. A non\-zero return code indicates a failure. 115 | .SS SEE ALSO 116 | .INDENT 0.0 117 | .IP \(bu 2 118 | \fB3rd\-party\-post\fP(1) 119 | .IP \(bu 2 120 | \fBmixer\-user\-bundler\fP(1) 121 | .IP \(bu 2 122 | \fBswupd\fP(1) 123 | .IP \(bu 2 124 | \fI\%https://github.com/clearlinux/swupd\-client\fP 125 | .IP \(bu 2 126 | \fI\%https://clearlinux.org/documentation/\fP 127 | .UNINDENT 128 | .SH COPYRIGHT 129 | (C) 2019 Intel Corporation, CC-BY-SA-3.0 130 | .\" Generated by docutils manpage writer. 131 | . 132 | -------------------------------------------------------------------------------- /swupd-wrapper/operations/update.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package operations 16 | 17 | import ( 18 | "bytes" 19 | "errors" 20 | "fmt" 21 | "io/ioutil" 22 | "log" 23 | "os/exec" 24 | "path" 25 | "path/filepath" 26 | "github.com/clearlinux/clr-user-bundles/cublib" 27 | ) 28 | 29 | func updateContent(statedir string, contentdir string, config cublib.TomlConfig) error { 30 | // -b and -N are essential, scripts are security dangerous since 3rd party content would get to run as root 31 | format, err := cublib.GetFormat() 32 | if err != nil { 33 | return err; 34 | } 35 | certPath := path.Join(contentdir, "/usr/share/clear/update-ca/Swupd_Root.pem") 36 | var cmd *exec.Cmd 37 | var out bytes.Buffer 38 | cmd = exec.Command("swupd", "update", "-b", "-N", "-F", format, "-S", statedir, "-p", contentdir, "-u", config.Bundle.URL, "-C", certPath) 39 | cmd.Stdout = &out 40 | cmd.Stderr = &out 41 | err = cmd.Run() 42 | if err != nil { 43 | return errors.New(out.String()) 44 | } 45 | newConfPath := "file://" + path.Join(contentdir, "usr", "user-config.toml") 46 | newConfig, err := cublib.GetConfig(newConfPath) 47 | if err != nil { 48 | return errors.New(fmt.Sprintf("Couldn't load new 3rd-party config: %s", err)) 49 | } 50 | if len(newConfig.Bundle.Includes) > 0 { 51 | cmd = exec.Command("swupd", append([]string{"bundle-add"}, newConfig.Bundle.Includes...)...) 52 | out = bytes.Buffer{} 53 | cmd.Stdout = &out 54 | cmd.Stderr = &out 55 | err = cmd.Run() 56 | if err != nil { 57 | return errors.New(fmt.Sprintf("Unable to install dependency bundle(s) %s to the base system: %s", newConfig.Bundle.Includes, out.String())) 58 | } 59 | } 60 | 61 | 62 | return nil 63 | } 64 | 65 | func Update(statedir string, contentdir string, skipPost bool) { 66 | // GetLock causes program exit on failure to acquire lockfile 67 | cublib.GetLock(statedir) 68 | defer cublib.ReleaseLock(statedir) 69 | pstatedir := path.Join(statedir, "3rd-party") 70 | chrootdir := path.Join(contentdir, "chroot") 71 | dlist, err := ioutil.ReadDir(chrootdir) 72 | if err != nil { 73 | log.Fatalf("Unable to read 3rd-party content directory (%s): %s", chrootdir, err) 74 | } 75 | 76 | for _, p := range dlist { 77 | // chroot dir should only be chroot directories and conf files so skip the conf files 78 | // as it is easier to make those names from the directory names 79 | if ext := filepath.Ext(p.Name()); ext != "" { 80 | continue 81 | } 82 | confPath := "file://" + path.Join(chrootdir, p.Name()) + ".toml" 83 | conf, err := cublib.GetConfig(confPath) 84 | if err != nil { 85 | log.Printf("WARNING: Unable to read 3rd party config (%s): %s", confPath, err) 86 | continue 87 | } 88 | // NOTE: content chroot exists but matching config doesn't => warning 89 | // BUT content chroot doesn't exist and config does => ignored, manual cleanup required 90 | if err = updateContent(path.Join(pstatedir, p.Name()), path.Join(chrootdir, p.Name()), conf); err != nil { 91 | log.Printf("WARNING: Unable to update (%s %s): %s", conf.Bundle.URL, conf.Bundle.Name, err) 92 | } 93 | } 94 | if skipPost { 95 | return 96 | } 97 | if err = cublib.PostProcess(statedir, contentdir); err != nil { 98 | log.Fatalf("%s", err) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /tests/swupd-wrapper/test-runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright © 2019 Intel Corporation 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | BASEDIR="$(pwd)" 18 | TESTDIR="$1" 19 | 20 | cleanup() { 21 | sudo rm -fr c c2 s o test.toml privatekey.pem Swupd_Root.pem 22 | if [ -f /etc/ca-certs/trusted/Swupd_Root.pem ]; then 23 | sudo clrtrust remove /etc/ca-certs/trusted/Swupd_Root.pem &> /dev/null 24 | fi 25 | popd &> /dev/null 26 | exit $1 27 | } 28 | 29 | pushd "$TESTDIR" &> /dev/null 30 | 31 | # Create test content chroots 32 | sudo cp -r content c 33 | sudo cp -r content2 c2 34 | 35 | # Generate template 36 | source /usr/lib/os-release 37 | sed -e "s|@VERSION@|${VERSION_ID}|" example-config.toml > test.toml 38 | sed -i "s|@TESTDIR@|file:///${PWD}/s/www/update|" test.toml 39 | 40 | # Build content 41 | sudo "${BASEDIR}/clr-user-bundles.py" s c test.toml 42 | if [ $? -ne 0 ]; then 43 | echo "Build content failed" 44 | cleanup 1 45 | fi 46 | 47 | # Install cert to trust store 48 | sudo clrtrust add Swupd_Root.pem &> /dev/null 49 | if [ $? -ne 0 ]; then 50 | echo "Certificate couldn't be added to trust store" 51 | cleanup 1 52 | fi 53 | 54 | # Install content 55 | sudo "${BASEDIR}/swupd-3rd-party" add "file:///${PWD}/s/www/update" -c "${PWD}/o" -p 56 | if [ $? -ne 0 ]; then 57 | echo "Install content failed" 58 | cleanup 1 59 | fi 60 | 61 | # Validate install worked 62 | diff o/chroot/*/usr/bin/test.sh c/usr/bin/test.sh 63 | if [ $? -ne 0 ]; then 64 | echo "Install content failed to verify" 65 | cleanup 1 66 | fi 67 | 68 | # Verify post process job didn't run automatically 69 | if [ -d "${PWD}/o/bin" ]; then 70 | echo "Post process job ran" 71 | cleanup 1 72 | fi 73 | 74 | # Run post process job manually 75 | sudo "${BASEDIR}/3rd-party-post" -c "${PWD}/o" 76 | if [ $? -ne 0 ]; then 77 | echo "Post process of update failed" 78 | cleanup 1 79 | fi 80 | 81 | # Verify post process job worked 82 | PATH="$PWD/o/bin:$PATH" test.sh | grep baz -q 83 | if [ $? -ne 0 ]; then 84 | echo "Post process job failed to verify" 85 | cleanup 1 86 | fi 87 | PATH=$PWD/o/bin:$PATH test.sh | grep fooenv -q 88 | if [ $? -ne 0 ]; then 89 | echo "Post process job failed to setup environment" 90 | cleanup 1 91 | fi 92 | 93 | # Build content2 94 | sudo "${BASEDIR}/clr-user-bundles.py" s c2 test.toml 95 | if [ $? -ne 0 ]; then 96 | echo "Build content2 failed" 97 | cleanup 1 98 | fi 99 | 100 | # Update to content2 101 | sudo "${BASEDIR}/swupd-3rd-party" update -c "${PWD}/o" 102 | if [ $? -ne 0 ]; then 103 | echo "Update to content2 failed" 104 | cleanup 1 105 | fi 106 | 107 | # Validate update worked 108 | diff o/chroot/*/usr/bin/test.sh c2/usr/bin/test.sh 109 | if [ $? -ne 0 ]; then 110 | echo "Update to content2 failed to verify" 111 | cleanup 1 112 | fi 113 | 114 | # Verify post process of update worked 115 | PATH="$PWD/o/bin:$PATH" test.sh | grep zab -q 116 | if [ $? -ne 0 ]; then 117 | echo "Post process of update failed to verify" 118 | cleanup 1 119 | fi 120 | PATH="$PWD/o/bin:$PATH" test.sh | grep barenv -q 121 | if [ $? -ne 0 ]; then 122 | echo "Post process of update failed to setup environment" 123 | cleanup 1 124 | fi 125 | 126 | # Verify remove works 127 | sudo "${BASEDIR}/swupd-3rd-party" remove "file:///${PWD}/s/www/update" test -c "${PWD}/o" 128 | if [ $? -ne 0 ]; then 129 | echo "Remove content failed" 130 | cleanup 1 131 | fi 132 | 133 | # Run post process on removal 134 | sudo "${BASEDIR}/3rd-party-post" -c "${PWD}/o" 135 | if [ $? -ne 0 ]; then 136 | echo "Post process of removal failed" 137 | cleanup 1 138 | fi 139 | 140 | # Verify post process of removal worked 141 | if [ -d o/bin ]; then 142 | echo "Post process of removal failed to verify" 143 | cleanup 1 144 | fi 145 | 146 | cleanup 0 147 | -------------------------------------------------------------------------------- /cublib/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cublib 16 | 17 | import ( 18 | "bytes" 19 | "encoding/base64" 20 | "io" 21 | "io/ioutil" 22 | "log" 23 | "net/http" 24 | "net/url" 25 | "os" 26 | "os/exec" 27 | "path" 28 | "strings" 29 | "syscall" 30 | ) 31 | 32 | var lockfd = -1 33 | 34 | func GetFormat() (string, error) { 35 | format, err := ioutil.ReadFile("/usr/share/defaults/swupd/format") 36 | if err != nil { 37 | return "", err 38 | } 39 | return string(format), nil 40 | } 41 | 42 | func GetVersion(uri string, statedir string) (string, error) { 43 | cmd := exec.Command("swupd", "update", "-S", statedir, "-s", "-u", uri) 44 | var out bytes.Buffer 45 | cmd.Stdout = &out 46 | err := cmd.Start() 47 | if err != nil { 48 | return "", err 49 | } 50 | if err = cmd.Wait(); err != nil { 51 | if exiterr, ok := err.(*exec.ExitError); ok { 52 | // swupd update -s will exit with 1 if it was successful so check that case 53 | if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { 54 | if status.ExitStatus() != 1 { 55 | return "", err 56 | } 57 | } 58 | } else { 59 | return "", err 60 | } 61 | } 62 | // Output is of the form: 63 | // Current OS version: XXX 64 | // Latest server version: YYY 65 | // ZZZZZZZZZZZ 66 | // 67 | // Grab YYY 68 | // TODO make a swupd-client check-update command to just give us this value. 69 | if _, err = out.ReadBytes(':'); err != nil { 70 | return "", err 71 | } 72 | if _, err = out.ReadBytes(':'); err != nil { 73 | return "", err 74 | } 75 | if _, err = out.ReadByte(); err != nil { 76 | return "", err 77 | } 78 | var version []byte 79 | if version, err = out.ReadBytes('\n'); err != nil { 80 | return "", err 81 | } 82 | return strings.TrimSpace(string(version)), nil 83 | } 84 | 85 | // Take lock for a given statedir, causes program to exit if it would fail to get the lock. 86 | func GetLock(statedir string) { 87 | if !path.IsAbs(statedir) { 88 | log.Fatalf("Error: state directory path (%s) is not absolute", statedir) 89 | } 90 | 91 | err := os.MkdirAll(statedir, 0700) 92 | if err != nil { 93 | log.Fatalf("Unable to create statedir (%s): %s", statedir, err) 94 | } 95 | 96 | lockfile := path.Join(statedir, "3rd-party.lock") 97 | flock := syscall.Flock_t{ 98 | Type: syscall.F_WRLCK, 99 | Start: 0, Len: 0, Whence: 0, Pid: int32(syscall.Getpid()), 100 | } 101 | fd, err := syscall.Open(lockfile, syscall.O_CREAT|syscall.O_RDWR|syscall.O_CLOEXEC, 0600) 102 | if err != nil { 103 | syscall.Close(fd) 104 | log.Fatalf("Lockfile (%s) open failed: %s", lockfile, err) 105 | } 106 | if err := syscall.FcntlFlock(uintptr(fd), syscall.F_SETLK, &flock); err != nil { 107 | log.Fatalf("Unable to set flock on %s: %s", lockfile, err) 108 | } 109 | lockfd = fd 110 | } 111 | 112 | func ReleaseLock(statedir string) { 113 | if lockfd >= 0 { 114 | syscall.Close(lockfd) 115 | } 116 | } 117 | 118 | func GetCert(statedir string, uri string) (string, error) { 119 | url, err := url.ParseRequestURI(uri) 120 | if err != nil { 121 | return "", err 122 | } 123 | var breader io.Reader 124 | if url.Scheme == "file" { 125 | buffer, err := ioutil.ReadFile(url.Path) 126 | if err != nil { 127 | return "", err 128 | } 129 | breader = bytes.NewReader(buffer) 130 | } else { 131 | resp, err := http.Get(uri) 132 | if err != nil { 133 | return "", err 134 | } else { 135 | defer resp.Body.Close() 136 | breader = resp.Body 137 | } 138 | } 139 | 140 | outpath := path.Join(statedir, "Swupd_Root.pem") 141 | out, err := os.Create(outpath) 142 | if err != nil { 143 | return "", err 144 | } 145 | if _, err = io.Copy(out, breader); err != nil { 146 | return "", err 147 | } 148 | 149 | return outpath, nil 150 | } 151 | 152 | func GetEncodedBundleName(url string, name string) string { 153 | return base64.StdEncoding.EncodeToString([]byte(url + name)) 154 | } 155 | -------------------------------------------------------------------------------- /mixer-user-bundler.1: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH MIXER-USER-BUNDLER 1 "" "" "" 4 | .SH NAME 5 | mixer-user-bundler \- 3rd-party bundle builder 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fBmixer\-user\-bundler [STATEDIR] [CHROOTDIR] [CONFIG] \fP 36 | .SH DESCRIPTION 37 | .sp 38 | \fBmixer\-user\-bundler\fP(1) is a software update generator that takes a content 39 | root (CHROOTDIR) and CONFIG to create update metadata consumable by \fBswupd\fP(1). 40 | .sp 41 | \fBmixer\-user\-bundler\fP will modify chrootdir with additional content required 42 | for \fBswupd\fP so it is recommended to make a copy of chrootdir that is used 43 | only with \fBmixer\-user\-bundler\fP\&. The STATEDIR content needs to be maintained 44 | between runs of \fBmixer\-user\-bundler\fP and the \fBswupd\fP update content is 45 | located within it at STATEDIR/www/update which needs to be available to clients 46 | to make use of this content. 47 | .sp 48 | The output of \fBmixer\-user\-bundler\fP is a set of manifests readable by \fBswupd\fP 49 | as well as all the OS content \fBswupd\fP needs to perform its update operations. 50 | The OS content includes all the files in an update as well as zero and 51 | delta\-packs for improved update performance. The content that 52 | \fBmixer\-user\-bundler\fP produces is tied to a specific format so that \fBswupd\fP 53 | is guaranteed to understand it if the client is using the right version of 54 | \fBswupd\fP\&. The way users are expected to interact with the content is through 55 | \fBswupd\-3rd\-party\fP(1) and \fB3rd\-party\-post\fP(1) commands that setup the 56 | content on the end users system based on the configuration provided when 57 | creating the user bundle. See \fB3rd\-party\-post\fP(1), \fBswupd\-3rd\-party\fP(1), 58 | \fBswupd\fP(1) and \fBos\-format\fP(7) for more details. 59 | .SH OPTIONS 60 | .sp 61 | The following options are applicable to be used to modify the core behavior and 62 | resources that \fBmixer\-user\-bundler\fP uses. 63 | .INDENT 0.0 64 | .IP \(bu 2 65 | \fB\-h, \-\-help\fP 66 | .sp 67 | Display general help information. 68 | .UNINDENT 69 | .SH FILES 70 | .sp 71 | \fICONFIG.toml\fP 72 | .INDENT 0.0 73 | .INDENT 3.5 74 | The mixer\-user\-bundler configuration file. This is a toml* formatted file 75 | containing two tables. The first is an upstream table with url and version 76 | keys, where url is a string that references the upstream content that 77 | the user bundle is based on and version is an integer of the upstream 78 | version the user bundle is to be built against. The second table is a 79 | bundle table with name, description, includes, url and bin keys. The name 80 | key is a string with the name of the user bundle being created, the 81 | description key is a short string describing what the bundle\(aqs purpose is 82 | for an end user, the includes key is an array of strings that are names 83 | of upstream bundles that are to be installed in order for the user bundle 84 | to function, the url key is a string which has the update url for the user 85 | bundle content (pointing to the STATEDIR/www/update content) and the bin 86 | key is an array of strings with absolute paths that resolve to executables 87 | in the CHROOTDIR that are to be exposed as executables to the end user. 88 | .UNINDENT 89 | .UNINDENT 90 | .SH EXIT STATUS 91 | .sp 92 | On success, 0 is returned. A non\-zero return code indicates a failure. 93 | .SS SEE ALSO 94 | .INDENT 0.0 95 | .IP \(bu 2 96 | \fB3rd\-party\-post\fP(1) 97 | .IP \(bu 2 98 | \fBswupd\fP(1) 99 | .IP \(bu 2 100 | \fBswupd\-3rd\-party\fP(1) 101 | .IP \(bu 2 102 | \fBos\-format\fP(7) 103 | .IP \(bu 2 104 | \fI\%https://github.com/clearlinux/swupd\-client\fP 105 | .IP \(bu 2 106 | \fI\%https://clearlinux.org/documentation/\fP 107 | .IP \(bu 2 108 | \fI\%https://github.com/toml\-lang/toml\fP 109 | .UNINDENT 110 | .SH COPYRIGHT 111 | (C) 2019 Intel Corporation, CC-BY-SA-3.0 112 | .\" Generated by docutils manpage writer. 113 | . 114 | -------------------------------------------------------------------------------- /swupd-wrapper/operations/add.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package operations 16 | 17 | import ( 18 | "bytes" 19 | "log" 20 | "path" 21 | "os" 22 | "os/exec" 23 | "github.com/clearlinux/clr-user-bundles/cublib" 24 | ) 25 | 26 | func Add(uri string, statedir string, contentdir string, skipPost bool) { 27 | // GetLock causes program exit on failure to acquire lockfile 28 | cublib.GetLock(statedir) 29 | defer cublib.ReleaseLock(statedir) 30 | format, err := cublib.GetFormat() 31 | if err != nil { 32 | log.Fatalf("Unable to get format from filesystem: %s", err) 33 | } 34 | version, err := cublib.GetVersion(uri, statedir) 35 | if err != nil { 36 | log.Fatalf("Unable to get version from uri (%s): %s", uri, err) 37 | } 38 | 39 | configBasename := "user-config.toml" 40 | postfix := path.Join("/", version, configBasename) 41 | configURI := uri + postfix 42 | config, err := cublib.GetConfig(configURI) 43 | if err != nil { 44 | log.Fatalf("Error accessing configuration from (%s): %s", configURI, err) 45 | } 46 | 47 | if config.Bundle.URL != uri { 48 | log.Printf("WARNING: bundle configured url (%s) and url used to add bundle (%s) differ", config.Bundle.URL, uri) 49 | } 50 | 51 | chrootdir := path.Join(contentdir, "chroot") 52 | err = os.MkdirAll(chrootdir, 0755) 53 | if err != nil { 54 | log.Fatalf("Unable to make toplevel 3rd party content directory (%s): %s", contentdir, err) 55 | } 56 | 57 | bnameEncoded := cublib.GetEncodedBundleName(config.Bundle.URL, config.Bundle.Name) 58 | pstatedir := path.Join(statedir, "3rd-party", bnameEncoded) 59 | err = os.MkdirAll(pstatedir, 0700) 60 | if err != nil { 61 | log.Fatalf("Unable to make 3rd party state directory (%s): %s", pstatedir, err) 62 | } 63 | configPath := path.Join(chrootdir, bnameEncoded) + ".toml" 64 | if _, err = os.Stat(configPath); !os.IsNotExist(err) { 65 | log.Fatalf("Config %s already exists, exiting", configPath) 66 | } 67 | err = cublib.WriteConfig(configPath, config, false) 68 | if err != nil { 69 | Remove(statedir, contentdir, config.Bundle.URL, config.Bundle.Name, false, false) 70 | log.Fatalf("Unable to save bundle configuration file to 3rd party state directory (%s): %s", pstatedir, err) 71 | } 72 | 73 | pchrootdir := path.Join(chrootdir, bnameEncoded) 74 | if _, err = os.Stat(pchrootdir); !os.IsNotExist(err) { 75 | log.Fatalf("Content path %s already exists, try running remove operation on partially installed content", pchrootdir) 76 | } 77 | err = os.MkdirAll(pchrootdir, 0755) 78 | if err != nil { 79 | Remove(statedir, contentdir, config.Bundle.URL, config.Bundle.Name, false, false) 80 | log.Fatalf("Unable to make 3rd party state directory (%s): %s", pstatedir, err) 81 | } 82 | 83 | certURI := config.Bundle.URL + path.Join("/", version, "Swupd_Root.pem") 84 | certPath, err := cublib.GetCert(pstatedir, certURI) 85 | if err != nil { 86 | Remove(statedir, contentdir, config.Bundle.URL, config.Bundle.Name, false, false) 87 | log.Fatalf("Unable to load certificate (%s): %s", certURI, err) 88 | } 89 | 90 | var cmd *exec.Cmd 91 | out := bytes.Buffer{} 92 | cmd = exec.Command("openssl", "verify", certPath) 93 | cmd.Stdout = &out 94 | cmd.Stderr = &out 95 | err = cmd.Run() 96 | if err != nil { 97 | Remove(statedir, contentdir, config.Bundle.URL, config.Bundle.Name, false, false) 98 | log.Printf("Certificate (%s) isn't trusted: %s", certURI, out.String()) 99 | log.Fatalf("Please add certificate to trust chain") 100 | } 101 | 102 | if len(config.Bundle.Includes) > 0 { 103 | cmd = exec.Command("swupd", append([]string{"bundle-add"}, config.Bundle.Includes...)...) 104 | out = bytes.Buffer{} 105 | cmd.Stdout = &out 106 | cmd.Stderr = &out 107 | err = cmd.Run() 108 | if err != nil { 109 | Remove(statedir, contentdir, config.Bundle.URL, config.Bundle.Name, false, false) 110 | log.Fatalf("Unable to install dependency bundle(s) %s to the base system: %s", config.Bundle.Includes, out.String()) 111 | } 112 | } 113 | 114 | // -b and -N are essential, scripts are security dangerous since 3rd party content would get to run as root 115 | cmd = exec.Command("swupd", "verify", "-f", "-b", "-N", "-S", pstatedir, "-p", pchrootdir, "-u", config.Bundle.URL, "-F", format, "-m", version, "-x", "-B", config.Bundle.Name, "-C", certPath) 116 | out = bytes.Buffer{} 117 | cmd.Stdout = &out 118 | cmd.Stderr = &out 119 | err = cmd.Run() 120 | if err != nil { 121 | Remove(statedir, contentdir, config.Bundle.URL, config.Bundle.Name, false, false) 122 | log.Fatalf("Unable to install bundle %s from %s: %s", config.Bundle.Name, config.Bundle.URL, out.String()) 123 | } 124 | if err = os.Remove(certPath); err != nil { 125 | log.Printf("WARNING: Unable to remove temporary cert (%s): %s", certPath, err) 126 | } 127 | 128 | if skipPost { 129 | return 130 | } 131 | if err = cublib.PostProcess(statedir, contentdir); err != nil { 132 | log.Fatalf("%s", err) 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /clr-user-bundles.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright © 2019 Intel Corporation 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from io import BytesIO 18 | 19 | import argparse 20 | import hashlib 21 | import os 22 | import shutil 23 | import stat 24 | import struct 25 | import subprocess 26 | import sys 27 | import tempfile 28 | import time 29 | 30 | import pycurl 31 | import toml 32 | 33 | ZERO_HASH = '0000000000000000000000000000000000000000000000000000000000000000' 34 | 35 | 36 | def parse_args(): 37 | """Handle arguments.""" 38 | p = argparse.ArgumentParser(description="Build Clear Linux user bundles.") 39 | p.add_argument('statedir', help="Directory to store user bundle creation output.") 40 | p.add_argument('chrootdir', help="Directory containing the content to be turned into a bundle.") 41 | p.add_argument('config', help="Configuration file for generating the user bundle.") 42 | return p.parse_args() 43 | 44 | 45 | def do_curl(url): 46 | """Perform a curl operation for `url`.""" 47 | c = pycurl.Curl() 48 | c.setopt(c.URL, url) 49 | c.setopt(c.FOLLOWLOCATION, True) 50 | c.setopt(c.FAILONERROR, True) 51 | buf = BytesIO() 52 | c.setopt(c.WRITEDATA, buf) 53 | try: 54 | c.perform() 55 | except pycurl.error as exptn: 56 | print(f"Unable to fetch {url}: {exptn}") 57 | return None 58 | finally: 59 | c.close() 60 | 61 | return buf.getvalue().decode('utf-8') 62 | 63 | 64 | def load_manifest(url, version, manifest_name): 65 | """Download and parse manifest.""" 66 | manifest_raw = do_curl(f"{url}/{version}/Manifest.{manifest_name}") 67 | manifest = {} 68 | if not manifest_raw: 69 | raise Exception(f"Unable to load manifest {manifest_name}") 70 | 71 | try: 72 | lines = manifest_raw.splitlines() 73 | for idx, line in enumerate(lines): 74 | content = line.split('\t') 75 | if content[0] == "MANIFEST": 76 | manifest['format'] = content[1] 77 | elif content[0] == "version:": 78 | manifest['version'] = content[1] 79 | elif content[0] == "previous:": 80 | manifest['previous'] = content[1] 81 | elif content[0] == "minversion:": 82 | manifest['minversin'] = content[1] 83 | elif content[0] == "filecount:": 84 | manifest['filecount'] = content[1] 85 | elif content[0] == "timestamp:": 86 | manifest['timestamp'] = content[1] 87 | elif content[0] == "contentsize:": 88 | manifest['contentsize'] = content[1] 89 | elif content[0] == "includes": 90 | if not manifest.get('includes'): 91 | manifest['includes'] = [] 92 | manifest['includes'].append(content[1]) 93 | elif len(content) == 4: 94 | if not manifest.get('files'): 95 | manifest['files'] = {} 96 | manifest['files'][content[3]] = content 97 | except Exception as _: 98 | raise Exception(f"Unable to parse manifest {manifest_name} at line {idx+1}: {line}") 99 | 100 | if not manifest.get('includes'): 101 | manifest['includes'] = [] 102 | if not manifest.get('files'): 103 | raise Exception(f"Invalid manifest {manifest_name}, missing file section") 104 | 105 | return manifest 106 | 107 | 108 | def copy_certificate(chrootdir, statedir, name): 109 | """Add certificate to chroot contents.""" 110 | if not os.path.isfile("privatekey.pem") or not os.path.isfile("Swupd_Root.pem"): 111 | subprocess.run(["openssl", "req", "-x509", "-sha256", "-nodes", "-newkey", "rsa:4096", 112 | "-keyout", "privatekey.pem", "-out", "Swupd_Root.pem", "-days", "1825", 113 | "-subj", "/C=US/ST=Oregon/L=Hillsboro/O=Example/CN=www.example.com"], 114 | capture_output=True, check=True) 115 | chroot_path = os.path.join(chrootdir, "usr", "share", "clear", "update-ca") 116 | os.makedirs(chroot_path, exist_ok=True) 117 | shutil.copyfile("Swupd_Root.pem", os.path.join(chroot_path, "Swupd_Root.pem")) 118 | 119 | 120 | def copy_config(full_config, chrootdir, statedir, version): 121 | """Add config to chroot contents.""" 122 | chroot_path = os.path.join(chrootdir, "usr") 123 | state_path = os.path.join(statedir, "www", "update", version) 124 | os.makedirs(chroot_path, exist_ok=True) 125 | os.makedirs(state_path, exist_ok=True) 126 | cpath = os.path.join(chroot_path, "user-config.toml") 127 | config = {} 128 | config['bundle'] = full_config['bundle'] 129 | with open(cpath, "w") as cfile: 130 | cfile.write(toml.dumps(config)) 131 | shutil.copyfile(cpath, os.path.join(state_path, "user-config.toml")) 132 | 133 | 134 | def get_base_manifests(includes, url, version, bundle): 135 | """Load upstream manifest files.""" 136 | manifests = {} 137 | mom = load_manifest(url, version, "MoM") 138 | if bundle: 139 | full_includes = [bundle] 140 | else: 141 | full_includes = includes if "os-core" in includes else includes + ["os-core"] 142 | 143 | for include in full_includes: 144 | try: 145 | include_version = mom['files'][include][2] 146 | except Exception as _: 147 | raise Exception(f"Included bundle {include} not found in upstream {url} for version {version}") 148 | manifests[include] = load_manifest(url, include_version, include) 149 | for recursive_include in manifests[include]['includes']: 150 | if manifests.get(recursive_include): 151 | continue 152 | try: 153 | rinclude_version = mom['files'][recursive_include][2] 154 | except Exception as _: 155 | raise Exception(f"Bundle {recursive_include}, included by {include} not found in upstream {url} for version {version}") 156 | manifests[recursive_include] = load_manifest(url, rinclude_version, recursive_include) 157 | 158 | return manifests 159 | 160 | 161 | def get_previous_version(statedir): 162 | """Parse version of previous user content release.""" 163 | version_path = os.path.join(statedir, "www", "update", "version") 164 | version = 0 165 | try: 166 | latest_format = sorted(os.listdir(version_path), key=lambda x: int(x[x.startswith("format") and len("format"):]))[-1] 167 | with open(os.path.join(version_path, f"{latest_format}", "latest"), "r") as lfile: 168 | version = int(lfile.read().strip()) 169 | except FileNotFoundError as exptn: 170 | pass 171 | return version 172 | 173 | 174 | def get_hash(path): 175 | """Get hash for the file contents.""" 176 | proc = subprocess.run(["swupd", "hashdump", "--quiet", path], encoding="utf-8", capture_output=True) 177 | if proc.returncode != 0: 178 | print(f"Failed to get hash of file {path}, swupd returned: {proc.stderr}, skipping") 179 | return None 180 | return proc.stdout.strip() 181 | 182 | 183 | def get_flags(path, cpath, lstat): 184 | """Get flags for the file.""" 185 | mode = list("....") 186 | if stat.S_ISDIR(lstat.st_mode): 187 | mode[0] = "D" 188 | elif stat.S_ISLNK(lstat.st_mode): 189 | mode[0] = "L" 190 | elif stat.S_ISREG(lstat.st_mode): 191 | mode[0] = "F" 192 | 193 | if mode[0] == ".": 194 | print(f"Invalide mode for {path}, skipping") 195 | return None 196 | 197 | if cpath.startswith("/usr/lib/kernel/") or cpath.startswith("/usr/lib/modules"): 198 | mode[2] = "b" 199 | 200 | return ''.join(mode) 201 | 202 | 203 | def add_metadata(chrootdir, url, manifest_format, version, bundle_name): 204 | """Create chroot config files.""" 205 | version_path = os.path.join(chrootdir, "usr", "lib") 206 | os.makedirs(version_path, exist_ok=True) 207 | with open(os.path.join(version_path, "os-release"), "w") as osfile: 208 | osfile.write(f"VERSION_ID={version}\n") 209 | sub_path = os.path.join(chrootdir, "usr", "share", "clear", "bundles") 210 | os.makedirs(sub_path, exist_ok=True) 211 | with open(os.path.join(sub_path, bundle_name), "w") as sfile: 212 | sfile.write("") 213 | # defaults_path = os.path.join(chrootdir, "usr", "share", "defaults", "swupd") 214 | # os.makedirs(defaults_path, exist_ok=True) 215 | # with open(os.path.join(defaults_path, "contenturl"), "w") as cfile: 216 | # cfile.write(f"{url}") 217 | # with open(os.path.join(defaults_path, "versionurl"), "w") as vfile: 218 | # vfile.write(f"{url}") 219 | # with open(os.path.join(defaults_path, "format"), "w") as ffile: 220 | # ffile.write(f"{manifest_format}") 221 | 222 | 223 | def scan_chroot(chrootdir, version, previous_version, previous_manifest, manifest_format): 224 | """Build manifest based off of a chroot directory.""" 225 | manifest = {} 226 | manifest['format'] = manifest_format 227 | manifest['version'] = version 228 | manifest['previous'] = previous_version 229 | manifest['filecount'] = 0 230 | manifest['timestamp'] = int(time.time()) 231 | manifest['contentsize'] = 0 232 | manifest['includes'] = [] 233 | manifest['files'] = {} 234 | for root, dirs, files in os.walk(chrootdir): 235 | for dname in dirs: 236 | entry = [] 237 | path = os.path.join(root, dname) 238 | cpath = path[path.startswith(chrootdir) and len(chrootdir):] 239 | dstat = os.lstat(path) 240 | dhash = get_hash(path) 241 | if not dhash: 242 | continue 243 | flags = get_flags(path, cpath, dstat) 244 | if not flags: 245 | continue 246 | version = manifest['version'] 247 | manifest['files'][cpath] = [flags, dhash, version, cpath] 248 | manifest['filecount'] += 1 249 | for fname in files: 250 | path = os.path.join(root, fname) 251 | cpath = path[path.startswith(chrootdir) and len(chrootdir):] 252 | fstat = os.lstat(path) 253 | fhash = get_hash(path) 254 | if not fhash: 255 | continue 256 | flags = get_flags(path, cpath, fstat) 257 | if not flags: 258 | continue 259 | version = manifest['version'] 260 | manifest['files'][cpath] = [flags, fhash, version, cpath] 261 | manifest['filecount'] += 1 262 | manifest['contentsize'] += os.lstat(path).st_size 263 | 264 | return manifest 265 | 266 | 267 | def combine_manifests(new_manifest, previous_manifest): 268 | """Create a combined manifest by modifying old and new manifest files.""" 269 | if not previous_manifest: 270 | return new_manifest 271 | 272 | for _, entry in new_manifest['files'].items(): 273 | # hash equal, use previous manifest entry's version 274 | if not previous_manifest['files'].get(entry[3]): 275 | continue 276 | if entry[1] == previous_manifest['files'][entry[3]][1]: 277 | entry[2] = previous_manifest['files'][entry[3]][2] 278 | 279 | return new_manifest 280 | 281 | 282 | def create_tar(input_path, output_path, input_name, output_name, transform=False, pack=False): 283 | """Create compressed tarfile in state directory.""" 284 | tar_name = f"{output_name}.tar" 285 | tar_path = os.path.join(output_path, tar_name) 286 | if pack: 287 | tar_cmd = f"tar -C {input_path} -cf {tar_path} -- {input_name[0]} {input_name[1]}" 288 | elif transform: 289 | tar_cmd = f"tar --no-recursion -C {input_path} -cf {tar_path} --transform s/{input_name}/{output_name}/ -- {input_name}" 290 | else: 291 | tar_cmd = f"tar --no-recursion -C {input_path} -cf {tar_path} -- {input_name}" 292 | proc = subprocess.run(tar_cmd, shell=True, capture_output=True) 293 | if proc.returncode != 0: 294 | raise Exception(f"Unable to create tar file for {os.path.join(input_path, input_name)} using:\n" 295 | f"{tar_cmd}\n") 296 | proc = subprocess.run(f"xz {tar_path}", shell=True, capture_output=True) 297 | if proc.returncode != 0: 298 | raise Exception(f"Unable to compress tar file for {os.path.join(input_path, input_name)}") 299 | os.rename(f"{tar_path}.xz", f"{tar_path}") 300 | 301 | 302 | def write_manifest(statedir, version, manifest, name): 303 | """Output final manifest file to the state directory.""" 304 | mname = f"Manifest.{name}" 305 | mtmp = [f"MANIFEST\t{manifest['format']}\n", 306 | f"version:\t{manifest['version']}\n", 307 | f"previous:\t{manifest['previous']}\n", 308 | f"filecount:\t{manifest['filecount']}\n", 309 | f"timestamp:\t{manifest['timestamp']}\n", 310 | f"contentsize:\t{manifest['contentsize']}\n"] 311 | for include in sorted(manifest['includes']): 312 | mtmp.append(f"includes:\t{include}\n") 313 | 314 | mtmp.append('\n') 315 | for key in sorted(manifest['files'], key=lambda k: (manifest['files'][k][2], 316 | manifest['files'][k][3])): 317 | mtmp.append(f"{manifest['files'][key][0]}\t{manifest['files'][key][1]}\t{manifest['files'][key][2]}\t{manifest['files'][key][3]}\n") 318 | 319 | out_path = os.path.join(statedir, "www", "update", version) 320 | if not os.path.isdir(out_path): 321 | os.makedirs(out_path, exist_ok=True) 322 | with open(os.path.join(out_path, mname), "w") as mout: 323 | mout.writelines(mtmp) 324 | create_tar(out_path, out_path, mname, mname) 325 | 326 | 327 | def write_fullfiles(statedir, chrootdir, version, manifest): 328 | """Output fullfiles content to the state directory.""" 329 | out_path = os.path.join(statedir, "www", "update", version, "files") 330 | if not os.path.isdir(out_path): 331 | os.makedirs(out_path, exist_ok=True) 332 | for val in manifest['files'].values(): 333 | if val[2] != version or val[1] == ZERO_HASH: 334 | continue 335 | out_file = os.path.join(out_path, val[1]) 336 | if os.path.isfile(f"{out_file}.tar"): 337 | continue 338 | in_file = os.path.join(chrootdir, val[3][1:]) 339 | create_tar(os.path.dirname(in_file), out_path, os.path.basename(in_file), val[1], True) 340 | 341 | 342 | def extract_file(statedir, file_entry, out_path): 343 | """Extract file in statedir to out_path.""" 344 | tar_file = os.path.join(statedir, "www", "update", file_entry[2], "files", f"{file_entry[1]}.tar") 345 | proc = subprocess.run(["tar", "-C", out_path, "-xf", tar_file], capture_output=True) 346 | if proc.returncode != 0: 347 | raise Exception("Unable to extract previously created fullfile {file_entry[3]} from version {file_entry[2]}") 348 | return os.path.join(out_path, file_entry[1]) 349 | 350 | 351 | def write_deltafiles(statedir, chrootdir, version, manifest, previous_manifest): 352 | """Output deltafiles content to the state directory.""" 353 | out_path = os.path.join(statedir, "www", "update", version, "delta") 354 | if not os.path.isdir(out_path): 355 | os.makedirs(out_path, exist_ok=True) 356 | for val in manifest['files'].values(): 357 | if val[2] != version or val[0] != "F..." or val[1] == ZERO_HASH: 358 | continue 359 | if not previous_manifest['files'].get(val[3]): 360 | continue 361 | pval = previous_manifest['files'][val[3]] 362 | out_file = os.path.join(out_path, f"{pval[2]}-{val[2]}-{pval[1]}-{val[1]}") 363 | if os.path.isfile(out_file): 364 | continue 365 | with tempfile.TemporaryDirectory(dir=os.getcwd()) as odir: 366 | previous_file = extract_file(statedir, pval, odir) 367 | current_file = os.path.join(chrootdir, val[3][1:]) 368 | try: 369 | proc = subprocess.run(["bsdiff", previous_file, current_file, out_file, "xz"], 370 | timeout=10, capture_output=True) 371 | if proc.returncode != 0: 372 | shutil.rmtree(out_file, ignore_errors=True) 373 | except subprocess.TimeoutExpired as exptn: 374 | shutil.rmtree(out_file, ignore_errors=True) 375 | 376 | 377 | def write_deltapack(statedir, chrootdir, version, manifest, previous_manifest, bundle_name): 378 | """Output deltapack to the statedir.""" 379 | out_path = os.path.join(statedir, "www", "update", version) 380 | delta_path = os.path.join(out_path, "delta") 381 | if not os.path.isdir(out_path): 382 | os.makedirs(out_path, exist_ok=True) 383 | with tempfile.TemporaryDirectory(dir=os.getcwd()) as odir: 384 | staged = os.path.join(odir, "staged") 385 | delta = os.path.join(odir, "delta") 386 | os.makedirs(staged) 387 | os.makedirs(delta) 388 | for val in manifest['files'].values(): 389 | if val[2] != version or val[1] == ZERO_HASH: 390 | continue 391 | delta_file = None 392 | if previous_manifest and previous_manifest['files'].get(val[3]): 393 | pval = previous_manifest['files'][val[3]] 394 | fname = f"{pval[2]}-{val[2]}-{pval[1]}-{val[1]}" 395 | delta_file = os.path.join(delta_path, fname) 396 | if not os.path.isfile(delta_file): 397 | delta_file = None 398 | copy_file = os.path.join(chrootdir, val[3][1:]) 399 | if delta_file: 400 | out_file = os.path.join(delta, fname) 401 | if os.path.isfile(f"{out_file}"): 402 | continue 403 | shutil.copyfile(delta_file, out_file) 404 | else: 405 | out_file = os.path.join(staged, val[1]) 406 | if os.path.exists(f"{out_file}"): 407 | continue 408 | extract_file(statedir, val, staged) 409 | if previous_manifest: 410 | out_file = f"pack-{bundle_name}-from-{previous_manifest['version']}" 411 | else: 412 | out_file = f"pack-{bundle_name}-from-0" 413 | create_tar(odir, out_path, (staged, delta), out_file, pack=True) 414 | 415 | 416 | def write_mom(statedir, manifest_format, version, previous_version, bundle_name): 417 | """Create a wrapper MoM for the user bundle.""" 418 | bundle_hash = get_hash(os.path.join(statedir, "www", "update", version, f"Manifest.{bundle_name}")) 419 | manifest = {'format': manifest_format, 420 | 'version': version, 421 | 'previous': previous_version, 422 | 'filecount': 1, 423 | 'timestamp': int(time.time()), 424 | 'contentsize': 0, 425 | 'includes': [], 426 | 'files': {bundle_name: ["M...", bundle_hash, version, bundle_name]}} 427 | write_manifest(statedir, version, manifest, "MoM") 428 | subprocess.run(["openssl", "smime", "-sign", "-binary", "-in", 429 | os.path.join(statedir, "www", "update", version, "Manifest.MoM"), 430 | "-signer", "Swupd_Root.pem", "-inkey", "privatekey.pem", 431 | "-outform", "DER", "-out", 432 | os.path.join(statedir, "www", "update", version, "Manifest.MoM.sig")], 433 | check=True) 434 | shutil.copyfile("Swupd_Root.pem", os.path.join(statedir, "www", "update", version, "Swupd_Root.pem")) 435 | 436 | 437 | def write_versions(statedir, manifest_format, version): 438 | """Create version files.""" 439 | out_path = os.path.join(statedir, "www", "update", "version", f"format{manifest_format}") 440 | os.makedirs(out_path, exist_ok=True) 441 | first_path = os.path.join(out_path, "first") 442 | latest_path = os.path.join(out_path, "latest") 443 | if not os.path.isfile(os.path.join(out_path, "first")): 444 | with open(first_path, "w") as fout: 445 | fout.write(f"{version}") 446 | with open(latest_path, "w") as lout: 447 | lout.write(f"{version}") 448 | 449 | 450 | def build_user_bundle(statedir, chrootdir, config): 451 | """Create manifests and other user bundle artifacts.""" 452 | base_version = config['upstream']['version'] 453 | bundle_includes = config['bundle']['includes'] 454 | bundle_name = config['bundle']['name'] 455 | bundle_url = config['bundle']['url'] 456 | included_manifests = get_base_manifests(bundle_includes, 457 | config['upstream']['url'], 458 | base_version, None) 459 | manifest_format = included_manifests['os-core']['format'] 460 | previous_version = get_previous_version(statedir) 461 | version = str(previous_version + 10) 462 | previous_version = str(previous_version) 463 | manifest_dir = os.path.join(os.getcwd(), statedir, "www", "update") 464 | if previous_version == "0": 465 | previous_manifest = None 466 | else: 467 | previous_manifest = get_base_manifests([bundle_name], f"file:///{manifest_dir}", previous_version, bundle_name)[bundle_name] 468 | copy_certificate(chrootdir, statedir, bundle_name) 469 | copy_config(config, chrootdir, statedir, version) 470 | add_metadata(chrootdir, bundle_url, manifest_format, version, bundle_name) 471 | new_manifest = scan_chroot(chrootdir, version, previous_version, previous_manifest, manifest_format) 472 | user_manifest = combine_manifests(new_manifest, previous_manifest) 473 | write_manifest(statedir, version, user_manifest, bundle_name) 474 | write_fullfiles(statedir, chrootdir, version, user_manifest) 475 | if previous_manifest: 476 | write_deltafiles(statedir, chrootdir, version, user_manifest, previous_manifest) 477 | write_deltapack(statedir, chrootdir, version, user_manifest, previous_manifest, bundle_name) 478 | write_mom(statedir, user_manifest['format'], version, previous_version, bundle_name) 479 | write_versions(statedir, user_manifest['format'], version) 480 | 481 | 482 | def main(): 483 | """Entry point for Clear Linux user bundle creation.""" 484 | args = parse_args() 485 | try: 486 | config = toml.load(args.config) 487 | except Exception as exptn: 488 | print(f"Unable to load configuration file: {exptn}") 489 | sys.exit(-1) 490 | 491 | try: 492 | build_user_bundle(args.statedir, args.chrootdir.rstrip("/"), config) 493 | except Exception as exptn: 494 | print(f"Unable to create user bundle: {exptn}") 495 | sys.exit(-1) 496 | 497 | if __name__ == '__main__': 498 | main() 499 | --------------------------------------------------------------------------------