[ Jun 30 14:45:40 Enabled. ]
[ Jun 30 14:45:40 Rereading configuration. ]
[ Jun 30 14:45:41 Rereading configuration. ]
[ Jun 30 14:45:44 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:45:44.653974525Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:45:44.657407103Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:45:44.657420803Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:45:44.677081795Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:45:44.753653182Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:04.774694528Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:63829"}
{"msg":"region b121ec99-9818-4659-8205-bf849a6c52e0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:04.798643875Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:04.803669895Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"uri":"/crucible/0/regions","method":"POST","req_id":"3aa902ca-06ba-4cbd-a026-174656d44178","remote_addr":"[fd00:1122:3344:101::b]:63829","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":18461,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:04.806747687Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible/regions/b121ec99-9818-4659-8205-bf849a6c52e0","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:04.816134784Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible/regions/b121ec99-9818-4659-8205-bf849a6c52e0","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:04.816195015Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b121ec99-9818-4659-8205-bf849a6c52e0\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b121ec99-9818-4659-8205-bf849a6c52e0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:04.89269751Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"region":"b121ec99-9818-4659-8205-bf849a6c52e0","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b121ec99-9818-4659-8205-bf849a6c52e0\" \"--data\" \"/data/regions/b121ec99-9818-4659-8205-bf849a6c52e0\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:04.895057397Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"region":"b121ec99-9818-4659-8205-bf849a6c52e0","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:04.956237607Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"uri":"/crucible/0/regions","method":"POST","req_id":"67dc9d09-16ae-4071-9d45-aee540347210","remote_addr":"[fd00:1122:3344:101::b]:63829","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.215917454Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"region":"b121ec99-9818-4659-8205-bf849a6c52e0","component":"worker"}
{"msg":"region b121ec99-9818-4659-8205-bf849a6c52e0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.216000186Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.216255151Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b121ec99-9818-4659-8205-bf849a6c52e0","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.21671165Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b121ec99-9818-4659-8205-bf849a6c52e0","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.222318191Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.236042935Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b121ec99-9818-4659-8205-bf849a6c52e0","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.239405212Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b121ec99-9818-4659-8205-bf849a6c52e0","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.239956733Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.240011054Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.240051285Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.240083755Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.24483999Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.249452732Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:46:05.697224909Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"uri":"/crucible/0/regions","method":"POST","req_id":"0aca5f78-86b8-4c73-a87f-6cc2705a7ae9","remote_addr":"[fd00:1122:3344:101::b]:63829","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":47,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.907699436Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:51859"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.911463391Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"uri":"/crucible/0/regions/b121ec99-9818-4659-8205-bf849a6c52e0","method":"GET","req_id":"eb66033c-1544-4110-a6f8-422d47c2714e","remote_addr":"[fd00:1122:3344:101::b]:51859","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":3710,"response_code":"200"}
{"msg":"path is \"/data/regions/b121ec99-9818-4659-8205-bf849a6c52e0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.914669404Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"datafile"}
{"msg":"dataset is oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible/regions/b121ec99-9818-4659-8205-bf849a6c52e0","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.923598453Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"datafile"}
{"msg":"snapshot is oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible/regions/b121ec99-9818-4659-8205-bf849a6c52e0@40227c7a-266d-46fa-a939-d7c07c77919f","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.931954139Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible/regions/b121ec99-9818-4659-8205-bf849a6c52e0\", \"40227c7a-266d-46fa-a939-d7c07c77919f\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.93199494Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.942392987Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"uri":"/crucible/0/regions/b121ec99-9818-4659-8205-bf849a6c52e0/snapshots/40227c7a-266d-46fa-a939-d7c07c77919f","method":"GET","req_id":"167a7795-92bd-4436-86b1-afde0631be20","remote_addr":"[fd00:1122:3344:101::b]:51859","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":27727,"response_code":"200"}
{"msg":"path is \"/data/regions/b121ec99-9818-4659-8205-bf849a6c52e0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.945729304Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"datafile"}
{"msg":"dataset is oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible/regions/b121ec99-9818-4659-8205-bf849a6c52e0","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.954661582Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"datafile"}
{"msg":"snapshot is oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible/regions/b121ec99-9818-4659-8205-bf849a6c52e0@40227c7a-266d-46fa-a939-d7c07c77919f","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.962490818Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible/regions/b121ec99-9818-4659-8205-bf849a6c52e0\", \"40227c7a-266d-46fa-a939-d7c07c77919f\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.962526339Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot b121ec99-9818-4659-8205-bf849a6c52e0-40227c7a-266d-46fa-a939-d7c07c77919f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.971824384Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"datafile"}
{"msg":"applying SMF actions for region b121ec99-9818-4659-8205-bf849a6c52e0 running snapshot 40227c7a-266d-46fa-a939-d7c07c77919f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.972694151Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.972714202Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"uri":"/crucible/0/regions/b121ec99-9818-4659-8205-bf849a6c52e0/snapshots/40227c7a-266d-46fa-a939-d7c07c77919f/run","method":"POST","req_id":"acb46712-191f-4840-9b40-cb1276cb292d","remote_addr":"[fd00:1122:3344:101::b]:51859","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":27014,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-b121ec99-9818-4659-8205-bf849a6c52e0-40227c7a-266d-46fa-a939-d7c07c77919f","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.988404405Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-b121ec99-9818-4659-8205-bf849a6c52e0-40227c7a-266d-46fa-a939-d7c07c77919f","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:01.999562647Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-b121ec99-9818-4659-8205-bf849a6c52e0-40227c7a-266d-46fa-a939-d7c07c77919f","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.016066796Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b121ec99-9818-4659-8205-bf849a6c52e0/.zfs/snapshot/40227c7a-266d-46fa-a939-d7c07c77919f","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.016408473Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.016471034Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.016519445Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.016566006Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.016628937Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.020357901Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.02428938Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"worker"}
{"msg":"region b121ec99-9818-4659-8205-bf849a6c52e0 running snapshot 40227c7a-266d-46fa-a939-d7c07c77919f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.024308Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"datafile"}
{"msg":"path is \"/data/regions/b121ec99-9818-4659-8205-bf849a6c52e0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.237171334Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"datafile"}
{"msg":"dataset is oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible/regions/b121ec99-9818-4659-8205-bf849a6c52e0","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.247238934Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"datafile"}
{"msg":"snapshot is oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible/regions/b121ec99-9818-4659-8205-bf849a6c52e0@40227c7a-266d-46fa-a939-d7c07c77919f","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.255437928Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_6cd13526-21d1-4797-8c85-06e921e76eaf/crucible/regions/b121ec99-9818-4659-8205-bf849a6c52e0\", \"40227c7a-266d-46fa-a939-d7c07c77919f\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.255480649Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-30T14:57:02.263724243Z","hostname":"oxz_crucible_ac7cdb90-2b18-4eb5-85e5-8831f84d3921","pid":29609,"uri":"/crucible/0/regions/b121ec99-9818-4659-8205-bf849a6c52e0/snapshots/40227c7a-266d-46fa-a939-d7c07c77919f/run","method":"POST","req_id":"69a1574c-f785-4a73-a65e-3fbcf7bcd11c","remote_addr":"[fd00:1122:3344:101::b]:51859","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":26557,"response_code":"200"}