[ Nov 22 20:31:48 Enabled. ]
[ Nov 22 20:31:48 Rereading configuration. ]
[ Nov 22 20:31:50 Rereading configuration. ]
[ Nov 22 20:31:53 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:31:53.821318415Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:31:53.824750149Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:31:53.824760199Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:31:53.838681729Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:31:53.939715246Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.012521741Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:40651"}
{"msg":"region b245043e-0ba2-46e9-a65f-d3efe2d80517 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.045877795Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.050719344Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.05546922Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"uri":"/crucible/0/regions","method":"POST","req_id":"a80c6b0d-3818-4227-b1ad-fdaaed1ba15c","remote_addr":"[fd00:1122:3344:101::a]:40651","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":15287,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.063581158Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.063606408Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b245043e-0ba2-46e9-a65f-d3efe2d80517\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.121804269Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"region":"b245043e-0ba2-46e9-a65f-d3efe2d80517","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b245043e-0ba2-46e9-a65f-d3efe2d80517\" \"--data\" \"/data/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.123749607Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"region":"b245043e-0ba2-46e9-a65f-d3efe2d80517","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.25328081Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"region":"b245043e-0ba2-46e9-a65f-d3efe2d80517","component":"worker"}
{"msg":"region b245043e-0ba2-46e9-a65f-d3efe2d80517 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.253327491Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.253429014Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b245043e-0ba2-46e9-a65f-d3efe2d80517","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.253921246Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b245043e-0ba2-46e9-a65f-d3efe2d80517","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.260001344Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.271543536Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b245043e-0ba2-46e9-a65f-d3efe2d80517","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.274435406Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.275045651Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.275106553Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.275152424Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.275186635Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.280553506Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.284353358Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:33:02.315421287Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"uri":"/crucible/0/regions","method":"POST","req_id":"32ef890b-5332-4240-9073-fa95ac3e98bb","remote_addr":"[fd00:1122:3344:101::a]:40651","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.0434397Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53527"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.046990207Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"uri":"/crucible/0/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517","method":"GET","req_id":"0ced350f-a414-47b9-8f93-7d00404da988","remote_addr":"[fd00:1122:3344:101::a]:53527","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":3466,"response_code":"200"}
{"msg":"path is \"/data/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.050439911Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"datafile"}
{"msg":"dataset is oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.060186279Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"datafile"}
{"msg":"snapshot is oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517@29084903-3e9b-48b6-ae11-4bd4b2e15f40","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.069016565Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517\", \"29084903-3e9b-48b6-ae11-4bd4b2e15f40\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.069050436Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.079945791Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"uri":"/crucible/0/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517/snapshots/29084903-3e9b-48b6-ae11-4bd4b2e15f40","method":"GET","req_id":"955dc06b-d019-495c-b999-42e7c10a9930","remote_addr":"[fd00:1122:3344:101::a]:53527","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":29519,"response_code":"200"}
{"msg":"path is \"/data/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.082902804Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"datafile"}
{"msg":"dataset is oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.090603162Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"datafile"}
{"msg":"snapshot is oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517@29084903-3e9b-48b6-ae11-4bd4b2e15f40","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.099842757Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517\", \"29084903-3e9b-48b6-ae11-4bd4b2e15f40\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.099881878Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot b245043e-0ba2-46e9-a65f-d3efe2d80517-29084903-3e9b-48b6-ae11-4bd4b2e15f40 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.109330899Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"datafile"}
{"msg":"applying SMF actions for region b245043e-0ba2-46e9-a65f-d3efe2d80517 running snapshot 29084903-3e9b-48b6-ae11-4bd4b2e15f40 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.109981875Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.110013686Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"uri":"/crucible/0/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517/snapshots/29084903-3e9b-48b6-ae11-4bd4b2e15f40/run","method":"POST","req_id":"b6e138c5-0477-494d-9cf2-1eda33ae1f51","remote_addr":"[fd00:1122:3344:101::a]:53527","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":27147,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-b245043e-0ba2-46e9-a65f-d3efe2d80517-29084903-3e9b-48b6-ae11-4bd4b2e15f40","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.119249161Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-b245043e-0ba2-46e9-a65f-d3efe2d80517-29084903-3e9b-48b6-ae11-4bd4b2e15f40","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.122821058Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-b245043e-0ba2-46e9-a65f-d3efe2d80517-29084903-3e9b-48b6-ae11-4bd4b2e15f40","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.135620261Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517/.zfs/snapshot/29084903-3e9b-48b6-ae11-4bd4b2e15f40","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.135850697Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.135899548Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.135940139Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.13597649Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.136021781Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.139537667Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.144274382Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"worker"}
{"msg":"region b245043e-0ba2-46e9-a65f-d3efe2d80517 running snapshot 29084903-3e9b-48b6-ae11-4bd4b2e15f40 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.144287763Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"datafile"}
{"msg":"path is \"/data/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.460319639Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"datafile"}
{"msg":"dataset is oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.469082544Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"datafile"}
{"msg":"snapshot is oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517@29084903-3e9b-48b6-ae11-4bd4b2e15f40","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.477194931Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_dcd1bfe3-9f4d-4a79-8ab7-1dc0c6900451/crucible/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517\", \"29084903-3e9b-48b6-ae11-4bd4b2e15f40\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.477230063Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-22T20:43:38.484687014Z","hostname":"oxz_crucible_7bfceb24-d461-40ce-8d0a-00e78380ff73","pid":26951,"uri":"/crucible/0/regions/b245043e-0ba2-46e9-a65f-d3efe2d80517/snapshots/29084903-3e9b-48b6-ae11-4bd4b2e15f40/run","method":"POST","req_id":"b7532d04-84bc-4ec5-ad84-19b126446f6a","remote_addr":"[fd00:1122:3344:101::a]:53527","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":24405,"response_code":"200"}