[ Dec  7 00:47:48 Enabled. ]
[ Dec  7 00:47:48 Rereading configuration. ]
[ Dec  7 00:47:53 Rereading configuration. ]
[ Dec  7 00:48:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_37290789-dec6-48b1-bd11-48c109977543/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_37290789-dec6-48b1-bd11-48c109977543/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:48:03.862241455Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:48:03.865433608Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:48:03.865443529Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:48:03.878797102Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:48:04.03695292Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.394933681Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33157"}
{"msg":"region 2be41bdf-6ad4-4d71-8f4b-e914fdee105e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.434486361Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.438783116Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.443407857Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"uri":"/crucible/0/regions","method":"POST","req_id":"cd00ca1a-6b60-4465-8c58-361e8be65394","remote_addr":"[fd00:1122:3344:101::a]:33157","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":14955,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_37290789-dec6-48b1-bd11-48c109977543/crucible/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.449617989Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_37290789-dec6-48b1-bd11-48c109977543/crucible/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.449633479Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2be41bdf-6ad4-4d71-8f4b-e914fdee105e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.506393129Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"region":"2be41bdf-6ad4-4d71-8f4b-e914fdee105e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2be41bdf-6ad4-4d71-8f4b-e914fdee105e\" \"--data\" \"/data/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.508723625Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"region":"2be41bdf-6ad4-4d71-8f4b-e914fdee105e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.69865366Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"uri":"/crucible/0/regions","method":"POST","req_id":"f094c73d-2348-450f-a466-8695e550777b","remote_addr":"[fd00:1122:3344:101::a]:33157","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.789838868Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"region":"2be41bdf-6ad4-4d71-8f4b-e914fdee105e","component":"worker"}
{"msg":"region 2be41bdf-6ad4-4d71-8f4b-e914fdee105e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.789888839Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.789981631Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2be41bdf-6ad4-4d71-8f4b-e914fdee105e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.790363658Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2be41bdf-6ad4-4d71-8f4b-e914fdee105e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.796338326Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.809154589Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2be41bdf-6ad4-4d71-8f4b-e914fdee105e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.811882853Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.812609947Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.812678318Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.812723999Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.81276336Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.816508414Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:19.820582054Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:49:20.005617593Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"uri":"/crucible/0/regions","method":"POST","req_id":"e5718873-eab5-4faf-9f89-15135827536f","remote_addr":"[fd00:1122:3344:101::a]:33157","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.791254391Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51135"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.794212619Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"uri":"/crucible/0/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e","method":"GET","req_id":"2c530e11-5962-47c6-a144-980c2c7148a6","remote_addr":"[fd00:1122:3344:101::a]:51135","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":2912,"response_code":"200"}
{"msg":"path is \"/data/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.798010925Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"datafile"}
{"msg":"dataset is oxp_37290789-dec6-48b1-bd11-48c109977543/crucible/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.806581063Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"datafile"}
{"msg":"snapshot is oxp_37290789-dec6-48b1-bd11-48c109977543/crucible/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e@063a0a88-ace1-4f78-9530-baa27bf1ea6e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.8150216Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_37290789-dec6-48b1-bd11-48c109977543/crucible/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e\", \"063a0a88-ace1-4f78-9530-baa27bf1ea6e\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.815062051Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.826276342Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"uri":"/crucible/0/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e/snapshots/063a0a88-ace1-4f78-9530-baa27bf1ea6e","method":"GET","req_id":"d37ee5b5-3916-4d15-9004-e0af1855cadc","remote_addr":"[fd00:1122:3344:101::a]:51135","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":28276,"response_code":"200"}
{"msg":"path is \"/data/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.829985235Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"datafile"}
{"msg":"dataset is oxp_37290789-dec6-48b1-bd11-48c109977543/crucible/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.83785287Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"datafile"}
{"msg":"snapshot is oxp_37290789-dec6-48b1-bd11-48c109977543/crucible/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e@063a0a88-ace1-4f78-9530-baa27bf1ea6e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.846304327Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_37290789-dec6-48b1-bd11-48c109977543/crucible/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e\", \"063a0a88-ace1-4f78-9530-baa27bf1ea6e\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.846329747Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 2be41bdf-6ad4-4d71-8f4b-e914fdee105e-063a0a88-ace1-4f78-9530-baa27bf1ea6e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.854763253Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"datafile"}
{"msg":"applying SMF actions for region 2be41bdf-6ad4-4d71-8f4b-e914fdee105e running snapshot 063a0a88-ace1-4f78-9530-baa27bf1ea6e (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.855562749Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.85558514Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"uri":"/crucible/0/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e/snapshots/063a0a88-ace1-4f78-9530-baa27bf1ea6e/run","method":"POST","req_id":"ccd6cf45-839f-4423-a7c3-47e0889e179a","remote_addr":"[fd00:1122:3344:101::a]:51135","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":25631,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-2be41bdf-6ad4-4d71-8f4b-e914fdee105e-063a0a88-ace1-4f78-9530-baa27bf1ea6e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.865579687Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-2be41bdf-6ad4-4d71-8f4b-e914fdee105e-063a0a88-ace1-4f78-9530-baa27bf1ea6e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.869624967Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-2be41bdf-6ad4-4d71-8f4b-e914fdee105e-063a0a88-ace1-4f78-9530-baa27bf1ea6e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.882412099Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e/.zfs/snapshot/063a0a88-ace1-4f78-9530-baa27bf1ea6e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.882702135Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.882754796Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.882799186Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.882838107Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.882885618Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.886304666Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.891526269Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"worker"}
{"msg":"region 2be41bdf-6ad4-4d71-8f4b-e914fdee105e running snapshot 063a0a88-ace1-4f78-9530-baa27bf1ea6e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:27.891542709Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"datafile"}
{"msg":"path is \"/data/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:28.173301585Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"datafile"}
{"msg":"dataset is oxp_37290789-dec6-48b1-bd11-48c109977543/crucible/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:28.182096958Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"datafile"}
{"msg":"snapshot is oxp_37290789-dec6-48b1-bd11-48c109977543/crucible/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e@063a0a88-ace1-4f78-9530-baa27bf1ea6e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:28.189740109Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_37290789-dec6-48b1-bd11-48c109977543/crucible/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e\", \"063a0a88-ace1-4f78-9530-baa27bf1ea6e\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:28.18976801Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-07T00:59:28.198228656Z","hostname":"oxz_crucible_71d96fc8-28af-43c3-9eb5-97ef5b28c882","pid":23082,"uri":"/crucible/0/regions/2be41bdf-6ad4-4d71-8f4b-e914fdee105e/snapshots/063a0a88-ace1-4f78-9530-baa27bf1ea6e/run","method":"POST","req_id":"4298365a-c3d8-472d-a6fc-8d50b72a237a","remote_addr":"[fd00:1122:3344:101::a]:51135","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":25548,"response_code":"200"}