Merge pull request #1 from johnbintz/slow-upgrades

Slow upgrades
This commit is contained in:
John Bintz 2020-06-13 18:04:15 -04:00 committed by GitHub
commit 8e24feb736
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 438 additions and 2468 deletions

2
.gitmodules vendored
View File

@ -1,3 +1,3 @@
[submodule "dokuwiki"] [submodule "dokuwiki"]
path = dokuwiki path = dokuwiki
url = git://github.com/ndarilek/dokuwiki.git url = https://github.com/splitbrain/dokuwiki.git

View File

@ -1,6 +1,8 @@
# -*- mode: ruby -*- # -*- mode: ruby -*-
# vi: set ft=ruby : # vi: set ft=ruby :
# CAUTION: DO NOT MAKE CHANGES TO THIS FILE. The vagrant-spk upgradevm process will overwrite it.
# Guess at a reasonable name for the VM based on the folder vagrant-spk is # Guess at a reasonable name for the VM based on the folder vagrant-spk is
# run from. The timestamp is there to avoid conflicts if you have multiple # run from. The timestamp is there to avoid conflicts if you have multiple
# folders with the same name. # folders with the same name.
@ -9,8 +11,18 @@ VM_NAME = File.basename(File.dirname(File.dirname(__FILE__))) + "_sandstorm_#{Ti
# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! # Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
VAGRANTFILE_API_VERSION = "2" VAGRANTFILE_API_VERSION = "2"
# ugly hack to prevent hashicorp's bitrot. See https://github.com/hashicorp/vagrant/issues/9442
# this setting is required for pre-2.0 vagrant, but causes an error as of 2.0.3,
# remove entirely when confident nobody uses vagrant 1.x for anything.
unless Vagrant::DEFAULT_SERVER_URL.frozen?
Vagrant::DEFAULT_SERVER_URL.replace('https://vagrantcloud.com')
end
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box = "sandstorm/debian-jessie64" # Base on the Sandstorm snapshots of the official Debian 9 (stretch) box with vboxsf support.
config.vm.box = "debian/contrib-stretch64"
config.vm.post_up_message = "Your virtual server is running at http://local.sandstorm.io:6090."
if Vagrant.has_plugin?("vagrant-vbguest") then if Vagrant.has_plugin?("vagrant-vbguest") then
# vagrant-vbguest is a Vagrant plugin that upgrades # vagrant-vbguest is a Vagrant plugin that upgrades
@ -21,10 +33,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.guest = "debian" config.vm.guest = "debian"
end end
# We forward port 6080, the Sandstorm web port, so that developers can # We forward port 6090, the vagrant-spk web port, so that developers can
# visit their sandstorm app from their browser as local.sandstorm.io:6080 # visit their Sandstorm app from their browser as local.sandstorm.io:6090
# (aka 127.0.0.1:6080). # (aka 127.0.0.1:6090).
config.vm.network :forwarded_port, guest: 6080, host: 6080 config.vm.network :forwarded_port, guest: 6090, host: 6090, host_ip: "127.0.0.1"
# Use a shell script to "provision" the box. This installs Sandstorm using # Use a shell script to "provision" the box. This installs Sandstorm using
# the bundled installer. # the bundled installer.
@ -49,12 +61,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
cpus = `nproc`.to_i cpus = `nproc`.to_i
total_kB_ram = `grep MemTotal /proc/meminfo | awk '{print $2}'`.to_i total_kB_ram = `grep MemTotal /proc/meminfo | awk '{print $2}'`.to_i
elsif host =~ /mingw/ elsif host =~ /mingw/
# powershell may not be available on Windows XP and Vista, so wrap this in a rescue block cpus = `powershell -Command "(Get-WmiObject Win32_Processor -Property NumberOfLogicalProcessors | Select-Object -Property NumberOfLogicalProcessors | Measure-Object NumberOfLogicalProcessors -Sum).Sum"`.to_i
begin total_kB_ram = `powershell -Command "[math]::Round((Get-WmiObject -Class Win32_ComputerSystem).TotalPhysicalMemory)"`.to_i / 1024
cpus = `powershell -Command "(Get-WmiObject Win32_Processor -Property NumberOfLogicalProcessors | Select-Object -Property NumberOfLogicalProcessors | Measure-Object NumberOfLogicalProcessors -Sum).Sum"`.to_i
total_kB_ram = `powershell -Command "Get-CimInstance -class cim_physicalmemory | % $_.Capacity}"`.to_i / 1024
rescue
end
end end
# Use the same number of CPUs within Vagrant as the system, with 1 # Use the same number of CPUs within Vagrant as the system, with 1
# as a default. # as a default.
@ -78,18 +86,25 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
vb.cpus = cpus vb.cpus = cpus
vb.memory = assign_ram_mb vb.memory = assign_ram_mb
vb.name = VM_NAME vb.name = VM_NAME
vb.customize ["modifyvm", :id, "--nictype1", "Am79C973"]
# /opt/app and /host-dot-sandstorm are used by vagrant-spk
override.vm.synced_folder "..", "/opt/app" override.vm.synced_folder "..", "/opt/app"
override.vm.synced_folder ENV["HOME"] + "/.sandstorm", "/host-dot-sandstorm" override.vm.synced_folder ENV["HOME"] + "/.sandstorm", "/host-dot-sandstorm"
override.vm.synced_folder "..", "/vagrant" # /vagrant is not used by vagrant-spk; we need this line so it gets disabled; if we removed the
# line, vagrant would automatically insert a synced folder in /vagrant, which is not what we want.
override.vm.synced_folder "..", "/vagrant", disabled: true
end end
config.vm.provider :libvirt do |libvirt, override| config.vm.provider :libvirt do |libvirt, override|
libvirt.cpus = cpus libvirt.cpus = cpus
libvirt.memory = assign_ram_mb libvirt.memory = assign_ram_mb
libvirt.default_prefix = VM_NAME libvirt.default_prefix = VM_NAME
# /opt/app and /host-dot-sandstorm are used by vagrant-spk
override.vm.synced_folder "..", "/opt/app", type: "9p", accessmode: "passthrough" override.vm.synced_folder "..", "/opt/app", type: "9p", accessmode: "passthrough"
override.vm.synced_folder ENV["HOME"] + "/.sandstorm", "/host-dot-sandstorm", type: "9p", accessmode: "passthrough" override.vm.synced_folder ENV["HOME"] + "/.sandstorm", "/host-dot-sandstorm", type: "9p", accessmode: "passthrough"
override.vm.synced_folder "..", "/vagrant", type: "9p", accessmode: "passthrough" # /vagrant is not used by vagrant-spk; we need this line so it gets disabled; if we removed the
# line, vagrant would automatically insert a synced folder in /vagrant, which is not what we want.
override.vm.synced_folder "..", "/vagrant", type: "9p", accessmode: "passthrough", disabled: true
end end
end end

2
.sandstorm/build.sh Normal file → Executable file
View File

@ -13,6 +13,8 @@ if [ -f /opt/app/dokuwiki/composer.json ] ; then
fi fi
rsync -a /opt/app/plugin/ /opt/app/dokuwiki/lib/plugins/sandstorm/ rsync -a /opt/app/plugin/ /opt/app/dokuwiki/lib/plugins/sandstorm/
cp /opt/app/500.html /opt/app/dokuwiki/
cp /opt/app/preload.php /opt/app/dokuwiki/inc/
for p in /opt/app/dokuwiki/{conf,data,lib/plugins,lib/tpl}; do for p in /opt/app/dokuwiki/{conf,data,lib/plugins,lib/tpl}; do
if [ ! -e $p.orig ]; then if [ ! -e $p.orig ]; then

27
.sandstorm/global-setup.sh Normal file → Executable file
View File

@ -1,21 +1,40 @@
#!/bin/bash #!/bin/bash
set -euo pipefail set -euo pipefail
# CAUTION: DO NOT MAKE CHANGES TO THIS FILE. The vagrant-spk upgradevm process will overwrite it.
# App-specific setup should be done in the setup.sh file.
# Set options for curl. Since we only want to show errors from these curl commands, we also use
# 'cat' to buffer the output; for more information:
# https://github.com/sandstorm-io/vagrant-spk/issues/158
CURL_OPTS="--silent --show-error" CURL_OPTS="--silent --show-error"
echo localhost > /etc/hostname echo localhost > /etc/hostname
hostname localhost hostname localhost
curl $CURL_OPTS https://install.sandstorm.io/ > /host-dot-sandstorm/caches/install.sh
# Grub updates don't silent install well
apt-mark hold grub-pc
apt-get update
apt-get upgrade -y
# Install curl that is needed below.
apt-get install -y curl
# The following line copies stderr through stderr to cat without accidentally leaving it in the
# output file. Be careful when changing. See: https://github.com/sandstorm-io/vagrant-spk/pull/159
curl $CURL_OPTS https://install.sandstorm.io/ 2>&1 > /host-dot-sandstorm/caches/install.sh | cat
SANDSTORM_CURRENT_VERSION=$(curl $CURL_OPTS -f "https://install.sandstorm.io/dev?from=0&type=install") SANDSTORM_CURRENT_VERSION=$(curl $CURL_OPTS -f "https://install.sandstorm.io/dev?from=0&type=install")
SANDSTORM_PACKAGE="sandstorm-$SANDSTORM_CURRENT_VERSION.tar.xz" SANDSTORM_PACKAGE="sandstorm-$SANDSTORM_CURRENT_VERSION.tar.xz"
if [[ ! -f /host-dot-sandstorm/caches/$SANDSTORM_PACKAGE ]] ; then if [[ ! -f /host-dot-sandstorm/caches/$SANDSTORM_PACKAGE ]] ; then
echo -n "Downloading Sandstorm version ${SANDSTORM_CURRENT_VERSION}..." echo -n "Downloading Sandstorm version ${SANDSTORM_CURRENT_VERSION}..."
curl $CURL_OPTS --output "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE.partial" "https://dl.sandstorm.io/$SANDSTORM_PACKAGE" curl $CURL_OPTS --output "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE.partial" "https://dl.sandstorm.io/$SANDSTORM_PACKAGE" 2>&1 | cat
mv "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE.partial" "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE" mv "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE.partial" "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE"
echo "...done." echo "...done."
fi fi
if [ ! -e /opt/sandstorm/latest/sandstorm ] ; then if [ ! -e /opt/sandstorm/latest/sandstorm ] ; then
echo -n "Installing Sandstorm version ${SANDSTORM_CURRENT_VERSION}..." echo -n "Installing Sandstorm version ${SANDSTORM_CURRENT_VERSION}..."
bash /host-dot-sandstorm/caches/install.sh -d -e "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE" >/dev/null bash /host-dot-sandstorm/caches/install.sh -d -e -p 6090 "/host-dot-sandstorm/caches/$SANDSTORM_PACKAGE" >/dev/null
echo "...done." echo "...done."
fi fi
modprobe ip_tables modprobe ip_tables
@ -32,3 +51,5 @@ GATEWAY_IP=$(ip route | grep ^default | cut -d ' ' -f 3)
if nc -z "$GATEWAY_IP" 3142 ; then if nc -z "$GATEWAY_IP" 3142 ; then
echo "Acquire::http::Proxy \"http://$GATEWAY_IP:3142\";" > /etc/apt/apt.conf.d/80httpproxy echo "Acquire::http::Proxy \"http://$GATEWAY_IP:3142\";" > /etc/apt/apt.conf.d/80httpproxy
fi fi
# Configure apt to retry fetching things that fail to download.
echo "APT::Acquire::Retries \"10\";" > /etc/apt/apt.conf.d/80sandstorm-retry

8
.sandstorm/launcher.sh Normal file → Executable file
View File

@ -2,7 +2,7 @@
# Create a bunch of folders under the clean /var that php and nginx expect to exist # Create a bunch of folders under the clean /var that php and nginx expect to exist
mkdir -p /var/lib/nginx mkdir -p /var/lib/nginx
mkdir -p /var/lib/php5/sessions mkdir -p /var/lib/php/sessions
mkdir -p /var/log mkdir -p /var/log
mkdir -p /var/log/nginx mkdir -p /var/log/nginx
mkdir -p /var/www mkdir -p /var/www
@ -12,10 +12,10 @@ rm -rf /var/run
mkdir -p /var/run mkdir -p /var/run
# Spawn php # Spawn php
/usr/sbin/php5-fpm --nodaemonize --fpm-config /etc/php5/fpm/php-fpm.conf & /usr/sbin/php-fpm7.0 -R --nodaemonize --fpm-config /etc/php/7.0/fpm/php-fpm.conf &
# Wait until php have bound its socket, indicating readiness # Wait until php have bound its socket, indicating readiness
while [ ! -e /var/run/php5-fpm.sock ] ; do while [ ! -e /var/run/php-fpm7.0.sock ] ; do
echo "waiting for php5-fpm to be available at /var/run/php5-fpm.sock" echo "waiting for php-fpm7.0 to be available at /var/run/php-fpm7.0.sock"
sleep .2 sleep .2
done done

Binary file not shown.

Binary file not shown.

File diff suppressed because it is too large Load Diff

View File

@ -9,7 +9,7 @@ const pkgdef :Spk.PackageDefinition = (
# The package definition. Note that the spk tool looks specifically for the # The package definition. Note that the spk tool looks specifically for the
# "pkgdef" constant. # "pkgdef" constant.
id = "nx3dqcmz2sjjz939vkg4847vvxrzqsatqfjrt3ea50z3jac5kv7h", id = "s97k5f7mkzjtz6nn440cm2dyv5ay4p3aqaxumhp1mv2s5k35cfah",
# Your app ID is actually its public key. The private key was placed in # Your app ID is actually its public key. The private key was placed in
# your keyring. All updates must be signed with the same key. # your keyring. All updates must be signed with the same key.
@ -19,9 +19,9 @@ const pkgdef :Spk.PackageDefinition = (
appTitle = (defaultText = "DokuWiki"), appTitle = (defaultText = "DokuWiki"),
appVersion = 5, # Increment this for every release. appVersion = 7, # Increment this for every release.
appMarketingVersion = (defaultText = "2017-02-19b-sandstorm6"), appMarketingVersion = (defaultText = "2017-02-19g~2020-06-13"),
# Human-readable representation of appVersion. Should match the way you # Human-readable representation of appVersion. Should match the way you
# identify versions of your app in documentation and marketing. # identify versions of your app in documentation and marketing.

View File

@ -21,7 +21,6 @@ http {
default_type application/octet-stream; default_type application/octet-stream;
# Logging # Logging
access_log off;
error_log stderr; error_log stderr;
# Prevent nginx from adding compression; this interacts badly with Sandstorm # Prevent nginx from adding compression; this interacts badly with Sandstorm
@ -38,6 +37,8 @@ http {
listen 8000 default_server; listen 8000 default_server;
listen [::]:8000 default_server ipv6only=on; listen [::]:8000 default_server ipv6only=on;
access_log /var/log/nginx/access.log;
# Allow arbitrarily large bodies - Sandstorm can handle them, and requests # Allow arbitrarily large bodies - Sandstorm can handle them, and requests
# are authenticated already, so there's no reason for apps to add additional # are authenticated already, so there's no reason for apps to add additional
# limits by default. # limits by default.
@ -47,6 +48,8 @@ http {
root /opt/app/dokuwiki; root /opt/app/dokuwiki;
index doku.php; index doku.php;
error_page 500 ../500.html;
location ~ /(data/|conf/|bin/|inc/|install.php) { deny all; } location ~ /(data/|conf/|bin/|inc/|install.php) { deny all; }
location / { location / {
@ -63,9 +66,10 @@ http {
location ~ \.php$ { location ~ \.php$ {
if (!-f $request_filename) { return 404; } if (!-f $request_filename) { return 404; }
include fastcgi_params; include fastcgi_params;
fastcgi_pass unix:/var/run/php5-fpm.sock; fastcgi_pass unix:/var/run/php-fpm7.0.sock;
fastcgi_param REDIRECT_STATUS 200; fastcgi_param REDIRECT_STATUS 200;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_intercept_errors on;
} }
} }

View File

@ -7,36 +7,39 @@ set -euo pipefail
export DEBIAN_FRONTEND=noninteractive export DEBIAN_FRONTEND=noninteractive
apt-get update apt-get update
apt-get install -y nginx php5-fpm php5-cli php5-curl git php5-dev libleveldb-dev apt-get install -y nginx php7.0-fpm php7.0-cli php7.0-curl git php7.0-dev libleveldb-dev
cd /usr/local/src cd /usr/local/src
git clone https://github.com/reeze/php-leveldb.git if [ ! -d php-leveldb ]; then
git clone https://github.com/reeze/php-leveldb.git
fi
cd php-leveldb cd php-leveldb
phpize phpize
./configure ./configure
make make
make install make install
cp /opt/app/leveldb.ini /etc/php5/mods-available cp /opt/app/leveldb.ini /etc/php/7.0/mods-available
php5enmod leveldb phpenmod leveldb
service nginx stop service nginx stop
service php5-fpm stop service php7.0-fpm stop
systemctl disable nginx systemctl disable nginx
systemctl disable php5-fpm systemctl disable php7.0-fpm
# patch /etc/php5/fpm/pool.d/www.conf to not change uid/gid to www-data # patch /etc/php/7.0/fpm/pool.d/www.conf to not change uid/gid to www-data
sed --in-place='' \ sed --in-place='' \
--expression='s/^listen.owner = www-data/#listen.owner = www-data/' \ --expression='s/^listen.owner = www-data/;listen.owner = www-data/' \
--expression='s/^listen.group = www-data/#listen.group = www-data/' \ --expression='s/^listen.group = www-data/;listen.group = www-data/' \
--expression='s/^user = www-data/#user = www-data/' \ --expression='s#^listen = .*#listen = /var/run/php-fpm7.0.sock#' \
--expression='s/^group = www-data/#group = www-data/' \ --expression='s/^user = www-data/;user = www-data/' \
/etc/php5/fpm/pool.d/www.conf --expression='s/^group = www-data/;group = www-data/' \
# patch /etc/php5/fpm/php-fpm.conf to not have a pidfile /etc/php/7.0/fpm/pool.d/www.conf
# patch /etc/php7.0/fpm/php-fpm.conf to not have a pidfile
sed --in-place='' \ sed --in-place='' \
--expression='s/^pid =/#pid =/' \ --expression='s/^pid =/;pid =/' \
/etc/php5/fpm/php-fpm.conf /etc/php/7.0/fpm/php-fpm.conf
# patch /etc/php5/fpm/pool.d/www.conf to no clear environment variables # patch /etc/php/7.0/fpm/pool.d/www.conf to no clear environment variables
# so we can pass in SANDSTORM=1 to apps # so we can pass in SANDSTORM=1 to apps
sed --in-place='' \ sed --in-place='' \
--expression='s/^;clear_env = no/clear_env=no/' \ --expression='s/^;clear_env = no/clear_env=no/' \
/etc/php5/fpm/pool.d/www.conf /etc/php/7.0/fpm/pool.d/www.conf
# Adjust fastcgi_params to use the patched fe_https # Adjust fastcgi_params to use the patched fe_https
sed --in-place='' \ sed --in-place='' \
--expression 's/^fastcgi_param *HTTPS.*$/fastcgi_param HTTPS \$fe_https if_not_empty;/' \ --expression 's/^fastcgi_param *HTTPS.*$/fastcgi_param HTTPS \$fe_https if_not_empty;/' \

41
500.html Normal file
View File

@ -0,0 +1,41 @@
<html>
<head><title>Uh oh, a plugin might be failing...</title></head>
<body>
<h1>Uh oh, a plugin might be failing...</h1>
<p>
You might have a plugin that's failing. If you installed any
custom plugins manually, they may not function correctly with
this version of DokuWiki. You should look in they
Grain Log for this grain to examine the errors from PHP-FPM.
If the issue is in plugin code, you'll have to remove the plugin manually
from the grain's filestructure via backup and restore.
You can then try installing a new version of the plugin, or find
a different one with better compatibility.
</p>
<h2>To fix a plugin issue:</h2>
<ol>
<li>Download a Backup of the grain.</li>
<li>Unzip the grain archive.</li>
<li>
Find the plugin that the error is referring to in
the <code>data/lib/dokiwiki/lib/plugins</code> folder and remove the
plugin's folder.
</li>
<li>
Zip up the unzipped grain archive, taking care to
preserve the same root files/folders as the original
Zip file.
</li>
<li>
Restore the grain from backup using the new Zip file.
</li>
<li>
Run the new grain. If it's working again, great!
If not, see if the error is still with plugins that
need removal/upgrading.
</li>
</ol>
</body>
</html>

40
README.md Normal file
View File

@ -0,0 +1,40 @@
# DokuWiki Sandstorm
Run [DokuWiki](https://www.dokuwiki.org/dokuwiki) on [Sandstorm](https://sandstorm.io/).
A lightweight wiki for your personal use.
## Details
* Runs in PHP 7 using PHP-FPM and an Nginx reverse proxy
* Uses Sandstorm authentication
## Troubleshooting
### PHP errors in the page
* See if the script referenced in the error is in `plugins`. You may have a
plugin that needs to be upgraded or disabled.
* If you can't disable/remove the plugin via the UI:
1. Download a backup of the grain
2. Unzip the backup into a new folder
3. Remove the plugin folder from the unzipped backup tree
4. Rezip up the backup tree, preserving the same base folders
5. Restore the grain from backup
## Building a new version
### Requirements
* git
* VirtualBox
* Vagrant
* [vagrant-spk](https://docs.sandstorm.io/en/latest/vagrant-spk/installation/)
### Setup & Commands
```
git submodule init
git submodule update
bin/upgrade_dokuwiki <target git tag from https://github.com/splitbrain/dokuwiki>
# follow the directions printed from the script
```

8
bin/cleanup_dokuwiki Executable file
View File

@ -0,0 +1,8 @@
#!/bin/bash
cd dokuwiki
git reset --hard
git clean -dxf
find . -name '*.orig' -exec rm -Rf {} \;
echo "Dokuwiki cleaned up for another build attempt"

22
bin/upgrade_dokuwiki Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
bin/cleanup_dokuwiki
cd dokuwiki
git fetch
git checkout $1
if [ $? -ne 0 ]; then
echo "Tag not found!"
exit 1
fi
cd ..
echo "Now:"
echo
echo "vagrant-spk vm destroy"
echo "vagrant-spk vm up"
echo "Test locally"
echo "vagrant-spk pack ../dokuwiki-sandstorm-$1.pkg"
echo "Test upgrade"
echo "vagrant-spk publish ../dokuwiki-sandstorm-$1.pkg"

@ -1 +1 @@
Subproject commit d3d13e525967aac5b41c37f6ed7b1a5dcb30fb78 Subproject commit a5690a8c536c2fc9bdb8871714c9978ea2c19c98

4
preload.php Normal file
View File

@ -0,0 +1,4 @@
<?php
ini_set('display_errors', '1');
ini_set('display_startup_errors', '1');
error_reporting(E_ALL);