Refactor utils

.githooks/pre-commit

@@ -5,14 +5,17 @@
 #
 # Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
 
-require_relative '../utils/helpers/copyrighted'
+require 'rubygems'
+require 'bundler/setup'
+require_relative '../utils/devtool/copyright'
 
 source_files_exts = ['.S', '.rs', '.rb']
 
-staged_files = `git --no-pager diff --name-only --cached`.split(/\n/)
-staged_files.each do |f|
-    if f.include?('Makefile') || source_files_exts.include?(File.extname(f))
-        puts "Checking for copyright range: #{f}"
-        exit 1 unless copyrighted?(f)
-    end
+staged_files = `git --no-pager diff --name-only --cached --diff-filter=d`.split(/\n/)
+staged_files.select! do |f|
+    f.include?('Makefile') ||
+        f.include?('Dockerfile') ||
+        source_files_exts.include?(File.extname(f))
 end
+
+exit(1) unless copyright_check_files(staged_files)

.github/workflows/build_rpi3.yml

@@ -35,6 +35,10 @@ jobs:
           rustup toolchain install nightly --component rust-src llvm-tools-preview
           rustup default nightly
           cargo install cargo-xbuild cargo-binutils
-    - name: Make all
+    - name: Set up Ruby
       run: |
-        ruby utils/make_all.rb rpi3
+        gem install bundler
+        bundle install --retry 3 --with development
+    - name: Make
+      run: |
+        bundle exec ruby utils/devtool.rb make rpi3

.github/workflows/build_rpi4.yml

@@ -35,6 +35,10 @@ jobs:
           rustup toolchain install nightly --component rust-src llvm-tools-preview
           rustup default nightly
           cargo install cargo-xbuild cargo-binutils
-    - name: Make all
+    - name: Set up Ruby
       run: |
-        ruby utils/make_all.rb rpi4
+        gem install bundler
+        bundle install --retry 3 --with development
+    - name: Make
+      run: |
+        bundle exec ruby utils/devtool.rb make rpi4

.github/workflows/check_fmt_clippy.yml

@@ -1,41 +0,0 @@
-name: Formatting + Clippy
-
-on:
-  push:
-    branches:
-      - master
-    paths-ignore:
-      - 'utils/**'
-      - 'doc/**'
-      - 'docker/**'
-  pull_request:
-    branches:
-      - master
-    paths-ignore:
-      - 'utils/**'
-      - 'doc/**'
-      - 'docker/**'
-
-jobs:
-  build:
-    name: Formatter and linter checks
-    runs-on: ubuntu-18.04
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up Ruby 2.x
-      uses: actions/setup-ruby@v1
-      with:
-        ruby-version: 2.x
-    - name: Set up Rust nightly
-      run: |
-          rustup self update
-          rustup toolchain install nightly --component rustfmt clippy rust-src
-          rustup default nightly
-          cargo install cargo-xbuild
-    - name: Run cargo fmt
-      run: |
-        ruby utils/fmt_all.rb --check
-    - name: Run cargo clippy
-      run: |
-        ruby utils/clippy_all.rb

.github/workflows/sanity.yml

@@ -0,0 +1,42 @@
+name: Various Sanity Checks
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    name: Various Sanity Checks
+    runs-on: ubuntu-18.04
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Ruby 2.x
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.x
+    - name: Set up Rust nightly
+      run: |
+          rustup self update
+          rustup toolchain install nightly --component rustfmt clippy rust-src
+          rustup default nightly
+          cargo install cargo-xbuild
+    - name: Set up Ruby
+      run: |
+        gem install bundler
+        bundle install --retry 3 --with development
+    - name: Setup misspell
+      run: |
+        curl -L -o ./install-misspell.sh https://git.io/misspell
+        sh ./install-misspell.sh -b ~/bin
+    - name: Run checks
+      run: |
+        bundle exec ruby utils/devtool.rb clippy
+        bundle exec ruby utils/devtool.rb copyright
+        bundle exec ruby utils/devtool.rb fmt_check
+        bundle exec ruby utils/devtool.rb misspell
+        bundle exec ruby utils/devtool.rb rubocop

.github/workflows/spelling.yml

@@ -1,23 +0,0 @@
-name: Spelling
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  build:
-    name: Spelling
-
-    runs-on: ubuntu-18.04
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: Install misspell and check
-      run: |
-        curl -L -o ./install-misspell.sh https://git.io/misspell
-        sh ./install-misspell.sh -b ~/bin
-        ~/bin/misspell .

.github/workflows/test_integration.yml

@@ -35,6 +35,10 @@ jobs:
           rustup toolchain install nightly --component rust-src llvm-tools-preview
           rustup default nightly
           cargo install cargo-xbuild cargo-binutils
+    - name: Set up Ruby
+      run: |
+        gem install bundler
+        bundle install --retry 3 --with development
     - name: Make all
       run: |
-        ruby utils/test_integration_all.rb
+        bundle exec ruby utils/devtool.rb test_integration

.github/workflows/test_unit.yml

@@ -35,6 +35,11 @@ jobs:
           rustup toolchain install nightly --component rust-src llvm-tools-preview
           rustup default nightly
           cargo install cargo-xbuild cargo-binutils
+    - name: Set up Ruby
+      run: |
+        gem install bundler
+        bundle install --retry 3 --with development
     - name: Make all
       run: |
-        ruby utils/test_unit_all.rb
+        bundle exec ruby utils/devtool.rb test_unit
+

.gitignore

@@ -1,3 +1,8 @@
+Gemfile.lock
 xbuild_sysroot
+
 **/target/*
 **/.gdb_history
+
+.bundle
+.vendor

.rubocop.yml

@@ -12,7 +12,7 @@
 Layout/IndentationWidth:
   Width: 4
 
-Metrics/LineLength:
+Layout/LineLength:
   Max: 100
 
 Metrics/MethodLength:

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "editor.formatOnSave": true,
+    "editor.rulers": [
+        100
+    ],
+}

Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+group :development do
+    gem 'colorize'
+    gem 'rubocop', require: false
+end

contributor_setup.sh

@@ -1,3 +1,4 @@
 #!/usr/bin/env bash
 
 git config core.hooksPath .githooks
+bundle install --path .vendor/bundle

utils/clean_all.rb

@@ -1,25 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# SPDX-License-Identifier: MIT OR Apache-2.0
-#
-# Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
-
-require 'fileutils'
-require_relative 'helpers/tutorial_folders.rb'
-
-def clean_all
-    crates = tutorial_folders
-
-    crates.each do |x|
-        x = File.dirname(x)
-        Dir.chdir(x) do
-            puts "Cleaning #{x}"
-            FileUtils.rm_rf('target')
-        end
-    end
-
-    FileUtils.rm_rf('xbuild_sysroot')
-end
-
-clean_all if $PROGRAM_NAME == __FILE__

utils/clippy_all.rb

@@ -1,26 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# SPDX-License-Identifier: MIT OR Apache-2.0
-#
-# Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
-
-require_relative 'helpers/tutorial_folders.rb'
-
-def clippy_all
-    crates = tutorial_folders
-
-    crates.each do |x|
-        x = File.dirname(x)
-
-        Dir.chdir(x) do
-            puts "Clippy: #{x}"
-            unless system('make clippy')
-                puts "\n\nClippy failed!"
-                exit(1) # Exit with error code
-            end
-        end
-    end
-end
-
-clippy_all if $PROGRAM_NAME == __FILE__

utils/devtool.rb

@@ -0,0 +1,215 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: MIT OR Apache-2.0
+#
+# Copyright (c) 2020 Andre Richter <andre.o.richter@gmail.com>
+
+require 'rubygems'
+require 'bundler/setup'
+require 'colorize'
+require 'fileutils'
+require_relative 'devtool/copyright'
+
+# Actions for tutorial folders
+class TutorialCrate
+    attr_reader :folder
+
+    def initialize(folder)
+        @folder = folder
+    end
+
+    def tutorial?
+        /[0-9]/.match?(@folder[0])
+    end
+
+    def clean
+        puts 'Cleaning '.light_blue + @folder
+
+        Dir.chdir(@folder) { FileUtils.rm_rf('target') }
+    end
+
+    def clippy
+        puts "Clippy #{@folder}".light_blue
+
+        Dir.chdir(@folder) { exit(1) unless system('make clippy') }
+    end
+
+    def fmt(args)
+        print 'Format '.light_blue
+        print "#{args} ".light_blue unless args.nil?
+        puts @folder
+
+        Dir.chdir(@folder) { exit(1) unless system("cargo fmt #{args}") }
+    end
+
+    def make(bsp)
+        puts "Make #{@folder} - BSP: #{bsp}".light_blue
+
+        Dir.chdir(@folder) { exit(1) unless system("BSP=#{bsp} make") }
+    end
+
+    def test_unit
+        return unless testable?
+
+        puts "Unit Tests #{@folder}".light_blue
+
+        Dir.chdir(@folder) { exit(1) unless system('TEST=unit make test') }
+    end
+
+    def test_integration
+        return unless testable?
+
+        puts "Integration Tests #{@folder}".light_blue
+
+        Dir.chdir(@folder) do
+            Dir['tests/*.rs'].sort.each do |t|
+                t = t.delete_prefix('tests/').delete_suffix('.rs')
+                exit(1) unless system("TEST=#{t} make test")
+            end
+        end
+    end
+
+    private
+
+    def testable?
+        Dir.exist?(@folder + '/tests')
+    end
+end
+
+# Forks commands to all applicable receivers
+class DevTool
+    def initialize
+        all = Dir['*/Cargo.toml'].sort
+
+        @crates = all.map { |c| TutorialCrate.new(c.delete_suffix('/Cargo.toml')) }
+    end
+
+    def clean
+        @crates.each(&:clean)
+        FileUtils.rm_rf('xbuild_sysroot')
+    end
+
+    def clippy
+        @crates.each do |c|
+            c.clippy
+            puts
+            puts
+        end
+    end
+
+    def diff
+        tuts = tutorials.map(&:folder)
+        padding = tuts.map(&:length).max
+
+        tuts[0..-2].each_with_index do |original, i|
+            update = tuts[i + 1]
+            diff_pair(original, update, padding)
+        end
+    end
+
+    def fmt(check = false)
+        args = '-- --check' if check
+
+        @crates.each { |c| c.fmt(args) }
+    end
+
+    def fmt_check
+        fmt(true)
+    end
+
+    def make(bsp = 'rpi3')
+        bsp = ARGV[1] if ARGV[1]
+
+        @crates.each do |c|
+            c.make(bsp)
+            puts
+            puts
+        end
+    end
+
+    def make_xtra
+        system('cd X1_JTAG_boot && bash update.sh')
+    end
+
+    def test_unit
+        @crates.each(&:test_unit)
+    end
+
+    def test_integration
+        @crates.each(&:test_integration)
+    end
+
+    def copyright
+        exit(1) unless copyright_check_files(copyright_source_files)
+    end
+
+    def misspell
+        exit(1) unless system("~/bin/misspell -error #{tracked_files.join(' ')}")
+    end
+
+    def rubocop
+        exit(1) unless system('rubocop')
+    end
+
+    def ready_for_publish
+        clean
+        fmt
+        misspell
+        rubocop
+        clippy
+        copyright
+
+        clean
+        make('rpi4')
+        make('rpi3')
+        make_xtra
+        test_unit
+        test_integration
+        diff
+        clean
+    end
+
+    private
+
+    def tutorials
+        @crates.select(&:tutorial?)
+    end
+
+    def tracked_files
+        `git ls-files`.split("\n")
+    end
+
+    def diff_pair(original, update, padding)
+        puts 'Diffing '.light_blue + original.ljust(padding) + " -> #{update}"
+        system("bash utils/diff_tut_folders.bash #{original} #{update}")
+    end
+
+    def copyright_source_files
+        extensions = ['.S', '.rs', '.rb']
+
+        files = tracked_files.select do |f|
+            f.include?('Makefile') ||
+                f.include?('Dockerfile') ||
+                extensions.include?(File.extname(f))
+        end
+
+        files
+    end
+end
+
+##--------------------------------------------------------------------------------------------------
+## Execution starts here
+##--------------------------------------------------------------------------------------------------
+tool = DevTool.new
+cmd = ARGV[0]
+commands = tool.public_methods(false).sort
+
+if !commands.include?(cmd&.to_sym)
+    puts "Usage: ./#{__FILE__} COMMAND"
+    puts
+    puts 'Commands:'
+    commands.each { |m| puts "  #{m}" }
+else
+    tool.public_send(cmd)
+end

utils/devtool/copyright.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: MIT OR Apache-2.0
+#
+# Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
+
+require 'colorize'
+
+def copyright_check_files(source_files)
+    source_files.sort.each do |f|
+        puts 'Checking for copyright: '.light_blue + f.to_s
+
+        years = copyright_years(f)
+        unless years.include?(Time.now.year)
+            puts "\tOnly found years: #{years}".red
+            return false
+        end
+    end
+
+    true
+end
+
+def copyright_years(file)
+    years = []
+    File.readlines(file).grep(/.*Copyright.*/).each do |x|
+        years << x.scan(/\d\d\d\d/).map!(&:to_i)
+    end
+
+    years.flatten
+end

utils/diff_all.rb

@@ -1,22 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# SPDX-License-Identifier: MIT OR Apache-2.0
-#
-# Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
-
-require 'fileutils'
-require_relative 'helpers/tutorial_folders.rb'
-
-def diff_all
-    crates = tutorial_folders(false)
-
-    (0..(crates.length - 2)).each do |i|
-        old = File.dirname(crates[i])
-        new = File.dirname(crates[i + 1])
-        puts "Diffing #{old} -> #{new}"
-        system("bash utils/helpers/diff_tut_folders.bash #{old} #{new}")
-    end
-end
-
-diff_all if $PROGRAM_NAME == __FILE__

utils/fmt_all.rb

@@ -1,34 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# SPDX-License-Identifier: MIT OR Apache-2.0
-#
-# Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
-
-require_relative 'helpers/tutorial_folders.rb'
-
-def fmt_all(check = false)
-    crates = tutorial_folders
-
-    args = if check != false
-               '-- --check'
-           else
-               ''
-           end
-
-    crates.each do |x|
-        x = File.dirname(x)
-        Dir.chdir(x) do
-            puts "Format #{x}"
-            unless system("cargo fmt #{args}")
-                puts "\n\nFmt check failed!"
-                exit(1) # Exit with error code
-            end
-        end
-    end
-end
-
-if $PROGRAM_NAME == __FILE__
-    # Any command line argument means --check
-    fmt_all(!ARGV[0].nil?)
-end

utils/helpers/copyrighted.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-# SPDX-License-Identifier: MIT OR Apache-2.0
-#
-# Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
-
-def copyrighted?(file)
-    copyright_years = []
-
-    File.readlines(file).grep(/.*Copyright.*/).each do |x|
-        copyright_years << x.scan(/\d\d\d\d/).map!(&:to_i)
-    end
-
-    copyright_years = copyright_years.flatten.uniq
-
-    unless copyright_years.include?(Time.now.year)
-        puts "\tHeader:   " + copyright_years.inspect
-        return false
-    end
-
-    true
-end

utils/helpers/tutorial_folders.rb

@@ -1,31 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# SPDX-License-Identifier: MIT OR Apache-2.0
-#
-# Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
-
-require 'fileutils'
-
-WITH_EXTRA = '[X0-9]'
-NO_EXTRA = '[0-9]'
-
-def tutorial_folders(with_extra = true, testable = false)
-    crates = Dir['*/Cargo.toml']
-
-    crates.delete_if do |x|
-        s = with_extra ? WITH_EXTRA : NO_EXTRA
-
-        !/[#{s}][0-9]/.match?(x[0..1])
-    end
-
-    if testable
-        crates.delete_if do |x|
-            x[0..1].to_i < 13
-        end
-    end
-
-    crates.sort!
-end
-
-puts tutorial_folders if $PROGRAM_NAME == __FILE__

utils/make_all.rb

@@ -1,25 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# SPDX-License-Identifier: MIT OR Apache-2.0
-#
-# Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
-
-require_relative 'helpers/tutorial_folders.rb'
-
-def make_all(bsp = 'rpi3')
-    crates = tutorial_folders
-
-    crates.each do |x|
-        x = File.dirname(x)
-        puts "\n\n" + x.to_s + "\n\n"
-        Dir.chdir(x) do
-            unless system("BSP=#{bsp} make")
-                puts "\n\nBuild failed!"
-                exit(1) # Exit with error code
-            end
-        end
-    end
-end
-
-make_all(ARGV[0]) if $PROGRAM_NAME == __FILE__

utils/ready_for_publish.rb

@@ -1,30 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# SPDX-License-Identifier: MIT OR Apache-2.0
-#
-# Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
-
-require_relative 'clean_all'
-require_relative 'clippy_all'
-require_relative 'diff_all'
-require_relative 'fmt_all'
-require_relative 'make_all'
-require_relative 'sanity_checks'
-require_relative 'test_integration_all'
-require_relative 'test_unit_all'
-
-clean_all
-fmt_all
-system('rubocop -l utils')
-sanity_checks
-clippy_all
-
-clean_all
-make_all
-test_unit_all
-test_integration_all
-system('cd X1_JTAG_boot && bash update.sh')
-diff_all
-clean_all
-system('~/bin/misspell .')

utils/sanity_checks.rb

@@ -1,41 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# SPDX-License-Identifier: MIT OR Apache-2.0
-#
-# Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
-
-require_relative 'helpers/copyrighted'
-require_relative 'helpers/tutorial_folders.rb'
-
-def patched?
-    crates = tutorial_folders
-
-    crates.each do |f|
-        unless File.readlines(f).grep(/patch.crates-io/).empty?
-            puts "#{fb} contains patch.crates-io!"
-            exit(1)
-        end
-    end
-end
-
-def check_old_copyrights
-    sources = Dir.glob('**/*.{S,rs,rb}') + Dir.glob('**/Makefile')
-
-    sources.delete_if do |x|
-        # if x is not in the index, treat this as an error
-        !system("git ls-files --error-unmatch #{x}", %i[out err] => File::NULL)
-    end
-
-    sources.sort.each do |f|
-        puts "Checking for copyright: #{f}"
-        exit(1) unless copyrighted?(f)
-    end
-end
-
-def sanity_checks
-    patched?
-    check_old_copyrights
-end
-
-sanity_checks if $PROGRAM_NAME == __FILE__

utils/test_integration_all.rb

@@ -1,30 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# SPDX-License-Identifier: MIT OR Apache-2.0
-#
-# Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
-
-require 'fileutils'
-require_relative 'helpers/tutorial_folders.rb'
-
-def run_tests
-    Dir['tests/*.rs'].sort.each do |int_test|
-        int_test = int_test.delete_prefix!('tests/').delete_suffix('.rs')
-        exit(1) unless system("TEST=#{int_test} make test")
-    end
-end
-
-def test_integration_all
-    crates = tutorial_folders(false, true)
-
-    crates.each do |x|
-        tut = File.dirname(x)
-        Dir.chdir(tut) do
-            puts "\n\n" + tut.to_s + "\n\n"
-            run_tests
-        end
-    end
-end
-
-test_integration_all if $PROGRAM_NAME == __FILE__

utils/test_unit_all.rb

@@ -1,23 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# SPDX-License-Identifier: MIT OR Apache-2.0
-#
-# Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
-
-require 'fileutils'
-require_relative 'helpers/tutorial_folders.rb'
-
-def test_unit_all
-    crates = tutorial_folders(false, true)
-
-    crates.each do |x|
-        x = File.dirname(x)
-        Dir.chdir(x) do
-            puts "\n\n" + x.to_s + "\n\n"
-            exit(1) unless system('TEST=unit make test')
-        end
-    end
-end
-
-test_unit_all if $PROGRAM_NAME == __FILE__