[ Dec 21 00:16:48 Enabled. ]
[ Dec 21 00:16:48 Rereading configuration. ]
[ Dec 21 00:16:50 Rereading configuration. ]
[ Dec 21 00:16:54 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_db606984-291b-40d5-964e-14a228d68e45/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_db606984-291b-40d5-964e-14a228d68e45/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:16:54.723446045Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:16:54.727303497Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:16:54.727315657Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:16:54.744623916Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:16:54.881580203Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.690709205Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49610"}
{"msg":"region dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.713587852Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.71392477Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.719540479Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"uri":"/crucible/0/regions","method":"POST","req_id":"dd67cc01-b1d9-4373-8ee0-b8eece91e88d","remote_addr":"[fd00:1122:3344:101::a]:49610","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":10078,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_db606984-291b-40d5-964e-14a228d68e45/crucible/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.725546818Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_db606984-291b-40d5-964e-14a228d68e45/crucible/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.725565469Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.777014522Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"region":"dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9\" \"--data\" \"/data/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.778992684Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"region":"dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.912142591Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"region":"dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","component":"worker"}
{"msg":"region dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.912208943Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.912295615Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.912688346Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.917206795Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.928370071Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.931121134Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.931668829Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.93172616Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.931766311Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.931802492Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.936027584Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.939910677Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:17:53.965015392Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"uri":"/crucible/0/regions","method":"POST","req_id":"69486c71-3a21-463c-9011-8a90c8d310e0","remote_addr":"[fd00:1122:3344:101::a]:49610","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.812129627Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64258"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.815163957Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"uri":"/crucible/0/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","method":"GET","req_id":"1745cc6e-2201-4156-9862-07e9eb23e7cc","remote_addr":"[fd00:1122:3344:101::a]:64258","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":2952,"response_code":"200"}
{"msg":"path is \"/data/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.818123146Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"datafile"}
{"msg":"dataset is oxp_db606984-291b-40d5-964e-14a228d68e45/crucible/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.827602946Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"datafile"}
{"msg":"snapshot is oxp_db606984-291b-40d5-964e-14a228d68e45/crucible/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9@9e83aa1a-7eee-477e-bf34-353a0eb8bb47","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.836821461Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_db606984-291b-40d5-964e-14a228d68e45/crucible/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9\", \"9e83aa1a-7eee-477e-bf34-353a0eb8bb47\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.836856881Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.848352566Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"uri":"/crucible/0/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9/snapshots/9e83aa1a-7eee-477e-bf34-353a0eb8bb47","method":"GET","req_id":"6cf1707f-68af-41f6-9fa3-9191b5c0c796","remote_addr":"[fd00:1122:3344:101::a]:64258","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":30242,"response_code":"200"}
{"msg":"path is \"/data/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.851786357Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"datafile"}
{"msg":"dataset is oxp_db606984-291b-40d5-964e-14a228d68e45/crucible/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.860354474Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"datafile"}
{"msg":"snapshot is oxp_db606984-291b-40d5-964e-14a228d68e45/crucible/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9@9e83aa1a-7eee-477e-bf34-353a0eb8bb47","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.869790544Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_db606984-291b-40d5-964e-14a228d68e45/crucible/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9\", \"9e83aa1a-7eee-477e-bf34-353a0eb8bb47\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.869823115Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9-9e83aa1a-7eee-477e-bf34-353a0eb8bb47 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.880036495Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"datafile"}
{"msg":"applying SMF actions for region dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9 running snapshot 9e83aa1a-7eee-477e-bf34-353a0eb8bb47 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.880652411Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.880674422Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"uri":"/crucible/0/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9/snapshots/9e83aa1a-7eee-477e-bf34-353a0eb8bb47/run","method":"POST","req_id":"52ac8828-3e4b-49c7-a353-12ec1abd1195","remote_addr":"[fd00:1122:3344:101::a]:64258","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":28920,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9-9e83aa1a-7eee-477e-bf34-353a0eb8bb47","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.892166356Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9-9e83aa1a-7eee-477e-bf34-353a0eb8bb47","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.896390828Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9-9e83aa1a-7eee-477e-bf34-353a0eb8bb47","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.908350245Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9/.zfs/snapshot/9e83aa1a-7eee-477e-bf34-353a0eb8bb47","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.90854711Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.908598902Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.908638893Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.908675334Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.908721995Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.912109585Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.916810609Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"worker"}
{"msg":"region dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9 running snapshot 9e83aa1a-7eee-477e-bf34-353a0eb8bb47 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:14.91682078Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"datafile"}
{"msg":"path is \"/data/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:15.113065308Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"datafile"}
{"msg":"dataset is oxp_db606984-291b-40d5-964e-14a228d68e45/crucible/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:15.122042116Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"datafile"}
{"msg":"snapshot is oxp_db606984-291b-40d5-964e-14a228d68e45/crucible/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9@9e83aa1a-7eee-477e-bf34-353a0eb8bb47","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:15.130589503Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_db606984-291b-40d5-964e-14a228d68e45/crucible/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9\", \"9e83aa1a-7eee-477e-bf34-353a0eb8bb47\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:15.130627954Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:28:15.139681713Z","hostname":"oxz_crucible_0e7abe31-ae21-4461-b74e-ed0e976f967e","pid":22730,"uri":"/crucible/0/regions/dde6d80d-ddf1-4fa5-b3f2-027cca69fdc9/snapshots/9e83aa1a-7eee-477e-bf34-353a0eb8bb47/run","method":"POST","req_id":"99af0c26-9170-4557-afa5-dd3a618b1a6b","remote_addr":"[fd00:1122:3344:101::a]:64258","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":27171,"response_code":"200"}