Merge pull request #1 from johnbintz/slow-upgrades

Slow upgrades

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "dokuwiki"]
 	path = dokuwiki
-	url = git://github.com/ndarilek/dokuwiki.git
+	url = https://github.com/splitbrain/dokuwiki.git

.sandstorm/Vagrantfile

@@ -1,6 +1,8 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
+# CAUTION: DO NOT MAKE CHANGES TO THIS FILE. The vagrant-spk upgradevm process will overwrite it.
+
 # Guess at a reasonable name for the VM based on the folder vagrant-spk is
 # run from.  The timestamp is there to avoid conflicts if you have multiple
 # folders with the same name.
@@ -9,8 +11,18 @@ VM_NAME = File.basename(File.dirname(File.dirname(__FILE__))) + "_sandstorm_#{Ti
 # Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
 VAGRANTFILE_API_VERSION = "2"
 
+# ugly hack to prevent hashicorp's bitrot. See https://github.com/hashicorp/vagrant/issues/9442
+# this setting is required for pre-2.0 vagrant, but causes an error as of 2.0.3,
+# remove entirely when confident nobody uses vagrant 1.x for anything.
+unless Vagrant::DEFAULT_SERVER_URL.frozen?
+  Vagrant::DEFAULT_SERVER_URL.replace('https://vagrantcloud.com')
+end
+
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box = "sandstorm/debian-jessie64"
+  # Base on the Sandstorm snapshots of the official Debian 9 (stretch) box with vboxsf support.
+  config.vm.box = "debian/contrib-stretch64"
+  config.vm.post_up_message = "Your virtual server is running at http://local.sandstorm.io:6090."
+
 
   if Vagrant.has_plugin?("vagrant-vbguest") then
     # vagrant-vbguest is a Vagrant plugin that upgrades
@@ -21,10 +33,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.guest = "debian"
   end
 
-  # We forward port 6080, the Sandstorm web port, so that developers can
+  # We forward port 6090, the vagrant-spk web port, so that developers can
-  # visit their sandstorm app from their browser as local.sandstorm.io:6080
+  # visit their Sandstorm app from their browser as local.sandstorm.io:6090
-  # (aka 127.0.0.1:6080).
+  # (aka 127.0.0.1:6090).
-  config.vm.network :forwarded_port, guest: 6080, host: 6080
+  config.vm.network :forwarded_port, guest: 6090, host: 6090, host_ip: "127.0.0.1"
 
   # Use a shell script to "provision" the box. This installs Sandstorm using
   # the bundled installer.
@@ -49,12 +61,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     cpus = `nproc`.to_i
     total_kB_ram = `grep MemTotal /proc/meminfo | awk '{print $2}'`.to_i
   elsif host =~ /mingw/
-    # powershell may not be available on Windows XP and Vista, so wrap this in a rescue block
+    cpus = `powershell -Command "(Get-WmiObject Win32_Processor -Property NumberOfLogicalProcessors | Select-Object -Property NumberOfLogicalProcessors | Measure-Object NumberOfLogicalProcessors -Sum).Sum"`.to_i
-    begin
+    total_kB_ram = `powershell -Command "[math]::Round((Get-WmiObject -Class Win32_ComputerSystem).TotalPhysicalMemory)"`.to_i / 1024
-      cpus = `powershell -Command "(Get-WmiObject Win32_Processor -Property NumberOfLogicalProcessors | Select-Object -Property NumberOfLogicalProcessors | Measure-Object NumberOfLogicalProcessors -Sum).Sum"`.to_i
-      total_kB_ram = `powershell -Command "Get-CimInstance -class cim_physicalmemory | % $_.Capacity}"`.to_i / 1024
-    rescue
-    end
   end
   # Use the same number of CPUs within Vagrant as the system, with 1
   # as a default.
@@ -78,18 +86,25 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     vb.cpus = cpus
     vb.memory = assign_ram_mb
     vb.name = VM_NAME
+    vb.customize ["modifyvm", :id, "--nictype1", "Am79C973"]
 
+    # /opt/app and /host-dot-sandstorm are used by vagrant-spk
     override.vm.synced_folder "..", "/opt/app"
     override.vm.synced_folder ENV["HOME"] + "/.sandstorm", "/host-dot-sandstorm"
-    override.vm.synced_folder "..", "/vagrant"
+    # /vagrant is not used by vagrant-spk; we need this line so it gets disabled; if we removed the
+    # line, vagrant would automatically insert a synced folder in /vagrant, which is not what we want.
+    override.vm.synced_folder "..", "/vagrant", disabled: true
   end
   config.vm.provider :libvirt do |libvirt, override|
     libvirt.cpus = cpus
     libvirt.memory = assign_ram_mb
     libvirt.default_prefix = VM_NAME
 
+    # /opt/app and /host-dot-sandstorm are used by vagrant-spk
     override.vm.synced_folder "..", "/opt/app", type: "9p", accessmode: "passthrough"
     override.vm.synced_folder ENV["HOME"] + "/.sandstorm", "/host-dot-sandstorm", type: "9p", accessmode: "passthrough"
-    override.vm.synced_folder "..", "/vagrant", type: "9p", accessmode: "passthrough"
+    # /vagrant is not used by vagrant-spk; we need this line so it gets disabled; if we removed the
+    # line, vagrant would automatically insert a synced folder in /vagrant, which is not what we want.
+    override.vm.synced_folder "..", "/vagrant", type: "9p", accessmode: "passthrough", disabled: true
   end
 end

.sandstorm/build.sh

@@ -13,6 +13,8 @@ if [ -f /opt/app/dokuwiki/composer.json ] ; then
 fi
 
 rsync -a /opt/app/plugin/ /opt/app/dokuwiki/lib/plugins/sandstorm/
+cp /opt/app/500.html /opt/app/dokuwiki/
+cp /opt/app/preload.php /opt/app/dokuwiki/inc/
 
 for p in /opt/app/dokuwiki/{conf,data,lib/plugins,lib/tpl}; do
   if [ ! -e $p.orig ]; then

.sandstorm/global-setup.sh

@@ -1,21 +1,40 @@
 #!/bin/bash
 set -euo pipefail
 
+# CAUTION: DO NOT MAKE CHANGES TO THIS FILE. The vagrant-spk upgradevm process will overwrite it.
+# App-specific setup should be done in the setup.sh file.
+
+# Set options for curl. Since we only want to show errors from these curl commands, we also use
+# 'cat' to buffer the output; for more information:
+# https://github.com/sandstorm-io/vagrant-spk/issues/158
+
 CURL_OPTS="--silent --show-error"
 echo localhost > /etc/hostname
 hostname localhost
-curl $CURL_OPTS https://install.sandstorm.io/ > /host-dot-sandstorm/caches/install.sh
+
+# Grub updates don't silent install well
+apt-mark hold grub-pc
+apt-get update
+apt-get upgrade -y
+
+# Install curl that is needed below.
+apt-get install -y curl
+
+# The following line copies stderr through stderr to cat without accidentally leaving it in the
+# output file. Be careful when changing. See: https://github.com/sandstorm-io/vagrant-spk/pull/159
+curl $CURL_OPTS https://install.sandstorm.io/ 2>&1 > /host-dot-sandstorm/caches/install.sh | cat
+
 SANDSTORM_CURRENT_VERSION=$(curl $CURL_OPTS -f "https://install.sandstorm.io/dev?from=0&type=install")
 SANDSTORM_PACKAGE="sandstorm-$SANDSTORM_CURRENT_VERSION.tar.xz"
 if [[ ! -f /host-dot-sandstorm/caches/$SANDSTORM_PACKAGE ]] ; then
     echo -n "Downloading Sandstorm version ${SANDSTORM_CURRENT_VERSION}..."
-    curl $CURL_OPTS --output "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE.partial" "https://dl.sandstorm.io/$SANDSTORM_PACKAGE"
+    curl $CURL_OPTS --output "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE.partial" "https://dl.sandstorm.io/$SANDSTORM_PACKAGE" 2>&1 | cat
     mv "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE.partial" "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE"
     echo "...done."
 fi
 if [ ! -e /opt/sandstorm/latest/sandstorm ] ; then
     echo -n "Installing Sandstorm version ${SANDSTORM_CURRENT_VERSION}..."
-    bash /host-dot-sandstorm/caches/install.sh -d -e "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE" >/dev/null
+    bash /host-dot-sandstorm/caches/install.sh -d -e -p 6090 "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE" >/dev/null
     echo "...done."
 fi
 modprobe ip_tables
@@ -32,3 +51,5 @@ GATEWAY_IP=$(ip route  | grep ^default  | cut -d ' ' -f 3)
 if nc -z "$GATEWAY_IP" 3142 ; then
     echo "Acquire::http::Proxy \"http://$GATEWAY_IP:3142\";" > /etc/apt/apt.conf.d/80httpproxy
 fi
+# Configure apt to retry fetching things that fail to download.
+echo "APT::Acquire::Retries \"10\";" > /etc/apt/apt.conf.d/80sandstorm-retry

.sandstorm/launcher.sh

@@ -2,7 +2,7 @@
 
 # Create a bunch of folders under the clean /var that php and nginx expect to exist
 mkdir -p /var/lib/nginx
-mkdir -p /var/lib/php5/sessions
+mkdir -p /var/lib/php/sessions
 mkdir -p /var/log
 mkdir -p /var/log/nginx
 mkdir -p /var/www
@@ -12,10 +12,10 @@ rm -rf /var/run
 mkdir -p /var/run
 
 # Spawn php
-/usr/sbin/php5-fpm --nodaemonize --fpm-config /etc/php5/fpm/php-fpm.conf &
+/usr/sbin/php-fpm7.0 -R --nodaemonize --fpm-config /etc/php/7.0/fpm/php-fpm.conf &
 # Wait until php have bound its socket, indicating readiness
-while [ ! -e /var/run/php5-fpm.sock ] ; do
+while [ ! -e /var/run/php-fpm7.0.sock ] ; do
-  echo "waiting for php5-fpm to be available at /var/run/php5-fpm.sock"
+  echo "waiting for php-fpm7.0 to be available at /var/run/php-fpm7.0.sock"
   sleep .2
 done
 

.sandstorm/sandstorm-pkgdef.capnp

@@ -9,7 +9,7 @@ const pkgdef :Spk.PackageDefinition = (
   # The package definition. Note that the spk tool looks specifically for the
   # "pkgdef" constant.
 
-  id = "nx3dqcmz2sjjz939vkg4847vvxrzqsatqfjrt3ea50z3jac5kv7h",
+  id = "s97k5f7mkzjtz6nn440cm2dyv5ay4p3aqaxumhp1mv2s5k35cfah",
   # Your app ID is actually its public key. The private key was placed in
   # your keyring. All updates must be signed with the same key.
 
@@ -19,9 +19,9 @@ const pkgdef :Spk.PackageDefinition = (
 
     appTitle = (defaultText = "DokuWiki"),
 
-    appVersion = 5,  # Increment this for every release.
+    appVersion = 7,  # Increment this for every release.
 
-    appMarketingVersion = (defaultText = "2017-02-19b-sandstorm6"),
+    appMarketingVersion = (defaultText = "2017-02-19g~2020-06-13"),
     # Human-readable representation of appVersion. Should match the way you
     # identify versions of your app in documentation and marketing.
 

.sandstorm/service-config/nginx.conf

@@ -21,7 +21,6 @@ http {
   default_type application/octet-stream;
 
   # Logging
-  access_log off;
   error_log stderr;
 
   # Prevent nginx from adding compression; this interacts badly with Sandstorm
@@ -38,6 +37,8 @@ http {
     listen 8000 default_server;
     listen [::]:8000 default_server ipv6only=on;
 
+    access_log /var/log/nginx/access.log;
+
     # Allow arbitrarily large bodies - Sandstorm can handle them, and requests
     # are authenticated already, so there's no reason for apps to add additional
     # limits by default.
@@ -47,6 +48,8 @@ http {
     root /opt/app/dokuwiki;
     index doku.php;
 
+    error_page 500 ../500.html;
+
     location ~ /(data/|conf/|bin/|inc/|install.php) { deny all; }
 
     location / {
@@ -63,9 +66,10 @@ http {
     location ~ \.php$ {
       if (!-f $request_filename) { return 404; }
       include fastcgi_params;
-      fastcgi_pass unix:/var/run/php5-fpm.sock;
+      fastcgi_pass unix:/var/run/php-fpm7.0.sock;
       fastcgi_param REDIRECT_STATUS 200;
       fastcgi_param  SCRIPT_FILENAME $document_root$fastcgi_script_name;
+      fastcgi_intercept_errors on;
     }
 
   }

.sandstorm/setup.sh

@@ -7,36 +7,39 @@ set -euo pipefail
 
 export DEBIAN_FRONTEND=noninteractive
 apt-get update
-apt-get install -y nginx php5-fpm php5-cli php5-curl git php5-dev libleveldb-dev
+apt-get install -y nginx php7.0-fpm php7.0-cli php7.0-curl git php7.0-dev libleveldb-dev
 cd /usr/local/src
-git clone https://github.com/reeze/php-leveldb.git
+if [ ! -d php-leveldb ]; then
+  git clone https://github.com/reeze/php-leveldb.git
+fi
 cd php-leveldb
 phpize
 ./configure
 make
 make install
-cp /opt/app/leveldb.ini /etc/php5/mods-available
+cp /opt/app/leveldb.ini /etc/php/7.0/mods-available
-php5enmod leveldb
+phpenmod leveldb
 service nginx stop
-service php5-fpm stop
+service php7.0-fpm stop
 systemctl disable nginx
-systemctl disable php5-fpm
+systemctl disable php7.0-fpm
-# patch /etc/php5/fpm/pool.d/www.conf to not change uid/gid to www-data
+# patch /etc/php/7.0/fpm/pool.d/www.conf to not change uid/gid to www-data
 sed --in-place='' \
-        --expression='s/^listen.owner = www-data/#listen.owner = www-data/' \
+        --expression='s/^listen.owner = www-data/;listen.owner = www-data/' \
-        --expression='s/^listen.group = www-data/#listen.group = www-data/' \
+        --expression='s/^listen.group = www-data/;listen.group = www-data/' \
-        --expression='s/^user = www-data/#user = www-data/' \
+        --expression='s#^listen = .*#listen = /var/run/php-fpm7.0.sock#' \
-        --expression='s/^group = www-data/#group = www-data/' \
+        --expression='s/^user = www-data/;user = www-data/' \
-        /etc/php5/fpm/pool.d/www.conf
+        --expression='s/^group = www-data/;group = www-data/' \
-# patch /etc/php5/fpm/php-fpm.conf to not have a pidfile
+        /etc/php/7.0/fpm/pool.d/www.conf
+# patch /etc/php7.0/fpm/php-fpm.conf to not have a pidfile
 sed --in-place='' \
-        --expression='s/^pid =/#pid =/' \
+        --expression='s/^pid =/;pid =/' \
-        /etc/php5/fpm/php-fpm.conf
+        /etc/php/7.0/fpm/php-fpm.conf
-# patch /etc/php5/fpm/pool.d/www.conf to no clear environment variables
+# patch /etc/php/7.0/fpm/pool.d/www.conf to no clear environment variables
 # so we can pass in SANDSTORM=1 to apps
 sed --in-place='' \
         --expression='s/^;clear_env = no/clear_env=no/' \
-        /etc/php5/fpm/pool.d/www.conf
+        /etc/php/7.0/fpm/pool.d/www.conf
 # Adjust fastcgi_params to use the patched fe_https
 sed --in-place='' \
         --expression 's/^fastcgi_param *HTTPS.*$/fastcgi_param  HTTPS               \$fe_https if_not_empty;/' \

500.html

@@ -0,0 +1,41 @@
+<html>
+  <head><title>Uh oh, a plugin might be failing...</title></head>
+  <body>
+    <h1>Uh oh, a plugin might be failing...</h1>
+    <p>
+      You might have a plugin that's failing. If you installed any
+      custom plugins manually, they may not function correctly with
+      this version of DokuWiki. You should look in they
+      Grain Log for this grain to examine the errors from PHP-FPM.
+      If the issue is in plugin code, you'll have to remove the plugin manually
+      from the grain's filestructure via backup and restore.
+      You can then try installing a new version of the plugin, or find
+      a different one with better compatibility.
+    </p>
+
+    <h2>To fix a plugin issue:</h2>
+
+    <ol>
+      <li>Download a Backup of the grain.</li>
+      <li>Unzip the grain archive.</li>
+      <li>
+        Find the plugin that the error is referring to in
+        the <code>data/lib/dokiwiki/lib/plugins</code> folder and remove the
+        plugin's folder.
+      </li>
+      <li>
+        Zip up the unzipped grain archive, taking care to
+        preserve the same root files/folders as the original
+        Zip file.
+      </li>
+      <li>
+        Restore the grain from backup using the new Zip file.
+      </li>
+      <li>
+        Run the new grain. If it's working again, great!
+        If not, see if the error is still with plugins that
+        need removal/upgrading.
+      </li>
+    </ol>
+  </body>
+</html>

README.md

@@ -0,0 +1,40 @@
+# DokuWiki Sandstorm
+
+Run [DokuWiki](https://www.dokuwiki.org/dokuwiki) on [Sandstorm](https://sandstorm.io/).
+A lightweight wiki for your personal use.
+
+## Details
+
+* Runs in PHP 7 using PHP-FPM and an Nginx reverse proxy
+* Uses Sandstorm authentication
+
+## Troubleshooting
+
+### PHP errors in the page
+
+* See if the script referenced in the error is in `plugins`. You may have a
+  plugin that needs to be upgraded or disabled.
+  * If you can't disable/remove the plugin via the UI:
+    1. Download a backup of the grain
+    2. Unzip the backup into a new folder
+    3. Remove the plugin folder from the unzipped backup tree
+    4. Rezip up the backup tree, preserving the same base folders
+    5. Restore the grain from backup
+
+## Building a new version
+
+### Requirements
+
+* git
+* VirtualBox
+* Vagrant
+* [vagrant-spk](https://docs.sandstorm.io/en/latest/vagrant-spk/installation/)
+
+### Setup & Commands
+
+```
+git submodule init
+git submodule update
+bin/upgrade_dokuwiki <target git tag from https://github.com/splitbrain/dokuwiki>
+# follow the directions printed from the script
+```

bin/cleanup_dokuwiki

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+cd dokuwiki
+git reset --hard
+git clean -dxf
+find . -name '*.orig' -exec rm -Rf {} \;
+
+echo "Dokuwiki cleaned up for another build attempt"

bin/upgrade_dokuwiki

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+bin/cleanup_dokuwiki
+cd dokuwiki
+git fetch
+git checkout $1
+if [ $? -ne 0 ]; then
+  echo "Tag not found!"
+  exit 1
+fi
+
+cd ..
+
+echo "Now:"
+echo
+echo "vagrant-spk vm destroy"
+echo "vagrant-spk vm up"
+echo "Test locally"
+echo "vagrant-spk pack ../dokuwiki-sandstorm-$1.pkg"
+echo "Test upgrade"
+echo "vagrant-spk publish ../dokuwiki-sandstorm-$1.pkg"
+

dokuwiki

@@ -1 +1 @@
-Subproject commit d3d13e525967aac5b41c37f6ed7b1a5dcb30fb78
+Subproject commit a5690a8c536c2fc9bdb8871714c9978ea2c19c98

preload.php

@@ -0,0 +1,4 @@
+<?php
+ini_set('display_errors', '1');
+ini_set('display_startup_errors', '1');
+error_reporting(E_ALL);