builder: fx-team-linux
slave: bld-linux64-spot-1079
starttime: 1469590465.56
results: exception (4)
buildid: 20160726203413
builduid: 61b716ac94bc4295b197f9b9c8593644
revision: 820cfffb1fe2b87a09d1c57d06d02107be8ac778

========= Started set props: master (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:25.563056) =========
master: http://buildbot-master91.bb.releng.usw2.mozilla.com:8001/
========= Finished set props: master (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:25.563661) =========

========= Started downloading to buildprops.json (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:25.563957) =========
========= Finished downloading to buildprops.json (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:25.575918) =========

========= Started 'rm -rf ...' (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:25.576195) =========
rm -rf scripts properties
 in dir /builds/slave/fx-team-lx-0000000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['rm', '-rf', 'scripts', 'properties']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-1079.build.releng.usw2.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-lx-0000000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.003414
========= master_lag: 0.05 =========
========= Finished 'rm -rf ...' (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:25.626089) =========

========= Started 'bash -c ...' (results: 0, elapsed: 12 secs) (at 2016-07-26 20:34:25.626433) =========
bash -c 'python /tools/checkouts/build-tools/buildfarm/utils/archiver_client.py mozharness --repo integration/fx-team --rev 820cfffb1fe2b87a09d1c57d06d02107be8ac778 --destination scripts --debug'
 in dir /builds/slave/fx-team-lx-0000000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['bash', '-c', u'python /tools/checkouts/build-tools/buildfarm/utils/archiver_client.py mozharness --repo integration/fx-team --rev 820cfffb1fe2b87a09d1c57d06d02107be8ac778 --destination scripts --debug']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-1079.build.releng.usw2.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-lx-0000000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
2016-07-26 20:34:25,669 truncating revision to first 12 chars
2016-07-26 20:34:25,669 Setting DEBUG logging.
2016-07-26 20:34:25,669 attempt 1/10
2016-07-26 20:34:25,669 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/integration/fx-team/820cfffb1fe2?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2016-07-26 20:34:26,534 attempt 1/10
2016-07-26 20:34:26,985 current task status: no status available at this point. state: STARTED
2016-07-26 20:34:26,986 sleeping for 11.00s (attempt 1/10)
2016-07-26 20:34:37,997 attempt 2/10
2016-07-26 20:34:38,204 current task status: Task completed! Check 's3_urls' for upload locations. state: SUCCESS
2016-07-26 20:34:38,284 unpacking tar archive at: fx-team-820cfffb1fe2/testing/mozharness/
program finished with exit code 0
elapsedTime=12.769255
========= master_lag: 0.01 =========
========= Finished 'bash -c ...' (results: 0, elapsed: 12 secs) (at 2016-07-26 20:34:38.405411) =========

========= Started set props: script_repo_revision (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:38.405707) =========
script_repo_revision: 820cfffb1fe2b87a09d1c57d06d02107be8ac778
========= Finished set props: script_repo_revision (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:38.406103) =========

========= Started downloading to oauth.txt (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:38.406362) =========
========= Finished downloading to oauth.txt (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:38.415451) =========

========= Started remove old nonce (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:38.415702) =========
rm -f nonce
 in dir /builds/slave/fx-team-lx-0000000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['rm', '-f', 'nonce']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-1079.build.releng.usw2.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-lx-0000000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.003054
========= master_lag: 0.04 =========
========= Finished remove old nonce (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:38.462870) =========

========= Started download_token exception (results: 4, elapsed: 45 mins, 7 secs) (at 2016-07-26 20:34:38.463162) =========
Slave: bld-linux64-spot-1079
IP: 10.132.53.161
Duration: 25200
URI: https://signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing3.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing3.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing3.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing3.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing3.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing7.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing7.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing7.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing7.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing7.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing1.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing1.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing1.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing1.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing1.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing5.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing5.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing5.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing5.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://signing5.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing2.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing2.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing2.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing2.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing2.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x4fb9ab8>: token generation failed, error message: 401 Authorization Required

<style type="text/css">
    p.error {
      color: red;
      font-family: Verdana, Arial, helvetica, sans-serif;
      font-weight: bold;
    }

    div {
      font-family: Verdana, Arial, helvetica, sans-serif;
    }

    div.stackTrace {
    }

    div.frame {
      padding: 1em;
      background: white;
      border-bottom: thin black dashed;
    }

    div.firstFrame {
      padding: 1em;
      background: white;
      border-top: thin black dashed;
      border-bottom: thin black dashed;
    }

    div.location {
    }

    div.snippet {
      margin-bottom: 0.5em;
      margin-left: 1em;
      background: #FFFFDD;
    }

    div.snippetHighlightLine {
      color: red;
    }

    span.code {
      font-family: "Courier New", courier, monotype;
    }

    span.function {
      font-weight: bold;
      font-family: "Courier New", courier, monotype;
    }

    table.variables {
      border-collapse: collapse;
      margin-left: 1em;
    }

    td.varName {
      vertical-align: top;
      font-weight: bold;
      padding-left: 0.5em;
      padding-right: 0.5em;
    }

    td.varValue {
      padding-left: 0.5em;
      padding-right: 0.5em;
    }

    div.variables {
      margin-bottom: 0.5em;
    }

    span.heading {
      font-weight: bold;
    }

    div.dict {
      background: #cccc99;
      padding: 2px;
      float: left;
    }

    td.dictKey {
      background: #ffff99;
      font-weight: bold;
    }

    td.dictValue {
      background: #ffff99;
    }

    div.list {
      background: #7777cc;
      padding: 2px;
      float: left;
    }

    div.listItem {
      background: #9999ff;
    }

    div.instance {
      background: #cc7777;
      padding: 2px;
      float: left;
    }

    span.instanceName {
      font-weight: bold;
      display: block;
    }

    span.instanceRepr {
      background: #ff9999;
      font-family: "Courier New", courier, monotype;
    }

    div.function {
      background: orange;
      font-weight: bold;
      float: left;
    }
</style>
<a href="#tbend">
<p class="error">&lt;type 'exceptions.Exception'&gt;: No more signing servers to try.</p>
</a><div class="stackTrace"></div><a name="tbend"> </a>
<p class="error">&lt;type 'exceptions.Exception'&gt;: No more signing servers to try.</p>
Traceback (most recent call last):
Failure: exceptions.Exception: No more signing servers to try.
========= Finished download_token exception (results: 4, elapsed: 45 mins, 7 secs) (at 2016-07-26 21:19:46.025180) =========

========= Skipped  (results: not started, elapsed: not started) =========
========= Skipped  (results: not started, elapsed: not started) =========
========= Started no change (results: 0, elapsed: 0 secs) (at 2016-07-26 21:19:46.026253) =========
bash -c 'for file in `ls -1`; do cat $file; done'
 in dir /builds/slave/fx-team-lx-0000000000000000000/properties (timeout 1200 secs)
 watching logfiles {}
 argv: ['bash', '-c', 'for file in `ls -1`; do cat $file; done']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-1079.build.releng.usw2.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-lx-0000000000000000000/properties
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.005193

========= master_lag: 0.04 =========
========= Finished no change (results: 0, elapsed: 0 secs) (at 2016-07-26 21:19:46.075456) =========

========= Started 'rm -f ...' (results: 0, elapsed: 0 secs) (at 2016-07-26 21:19:46.075768) =========
rm -f oauth.txt
 in dir /builds/slave/fx-team-lx-0000000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['rm', '-f', 'oauth.txt']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-1079.build.releng.usw2.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-lx-0000000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.002574
========= master_lag: 0.04 =========
========= Finished 'rm -f ...' (results: 0, elapsed: 0 secs) (at 2016-07-26 21:19:46.123186) =========

========= Started reboot skipped (results: 3, elapsed: 0 secs) (at 2016-07-26 21:19:46.123488) =========
========= Finished reboot skipped (results: 3, elapsed: 0 secs) (at 2016-07-26 21:19:46.123752) =========

========= Total master_lag: 0.19 =========