[ Feb 22 18:31:16 Enabled. ]
[ Feb 22 18:31:16 Rereading configuration. ]
[ Feb 22 18:31:18 Rereading configuration. ]
[ Feb 22 18:31:27 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:27.101830269Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:27.105885538Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:27.105898378Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:27.319541245Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.014533015Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:41583"}
{"msg":"region 9f852063-217e-476c-be0f-1cdbe52b6721 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.029868244Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.03014871Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"uri":"/crucible/0/regions","method":"POST","req_id":"31624d3a-721a-469e-8c29-dabbba57275f","remote_addr":"[fd00:1122:3344:101::a]:41583","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":5001,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.030730971Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/9f852063-217e-476c-be0f-1cdbe52b6721","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.039760017Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/9f852063-217e-476c-be0f-1cdbe52b6721","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.039790658Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9f852063-217e-476c-be0f-1cdbe52b6721\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/9f852063-217e-476c-be0f-1cdbe52b6721\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.097523453Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"region":"9f852063-217e-476c-be0f-1cdbe52b6721","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.163739095Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"uri":"/crucible/0/regions","method":"POST","req_id":"f2325b57-fe89-42f0-976f-4f70c35c8b43","remote_addr":"[fd00:1122:3344:101::a]:41583","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":675,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.231864383Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"region":"9f852063-217e-476c-be0f-1cdbe52b6721","component":"worker"}
{"msg":"region 9f852063-217e-476c-be0f-1cdbe52b6721 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.231902164Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.232021126Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9f852063-217e-476c-be0f-1cdbe52b6721","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.232491436Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9f852063-217e-476c-be0f-1cdbe52b6721","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.236542615Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.251475586Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9f852063-217e-476c-be0f-1cdbe52b6721","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.254304121Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9f852063-217e-476c-be0f-1cdbe52b6721","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.256736708Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.258727147Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.258775488Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.258815189Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.264336167Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.268170041Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.515214189Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"uri":"/crucible/0/regions","method":"POST","req_id":"597286ce-9cd1-454b-9b66-285cf05c0b65","remote_addr":"[fd00:1122:3344:101::a]:41583","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":103,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.120652334Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51629"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.12348336Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"uri":"/crucible/0/regions/9f852063-217e-476c-be0f-1cdbe52b6721","method":"GET","req_id":"d29caa86-94a1-451a-b2dd-1cafd7538c9f","remote_addr":"[fd00:1122:3344:101::a]:51629","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":2804,"response_code":"200"}
{"msg":"path is \"/data/regions/9f852063-217e-476c-be0f-1cdbe52b6721\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.123661483Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"datafile"}
{"msg":"dataset is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/9f852063-217e-476c-be0f-1cdbe52b6721","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.13424935Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"datafile"}
{"msg":"snapshot is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/9f852063-217e-476c-be0f-1cdbe52b6721@5d9df203-ce89-4f7f-9ce1-08b8316a4558","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.144463469Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/9f852063-217e-476c-be0f-1cdbe52b6721\", \"5d9df203-ce89-4f7f-9ce1-08b8316a4558\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.14450702Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.156182817Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"uri":"/crucible/0/regions/9f852063-217e-476c-be0f-1cdbe52b6721/snapshots/5d9df203-ce89-4f7f-9ce1-08b8316a4558","method":"GET","req_id":"4e71034e-9e3d-4afb-9b51-20118d328d60","remote_addr":"[fd00:1122:3344:101::a]:51629","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":32542,"response_code":"200"}
{"msg":"path is \"/data/regions/9f852063-217e-476c-be0f-1cdbe52b6721\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.156445003Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"datafile"}
{"msg":"dataset is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/9f852063-217e-476c-be0f-1cdbe52b6721","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.165229494Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"datafile"}
{"msg":"snapshot is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/9f852063-217e-476c-be0f-1cdbe52b6721@5d9df203-ce89-4f7f-9ce1-08b8316a4558","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.173488965Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/9f852063-217e-476c-be0f-1cdbe52b6721\", \"5d9df203-ce89-4f7f-9ce1-08b8316a4558\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.173525076Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 9f852063-217e-476c-be0f-1cdbe52b6721-5d9df203-ce89-4f7f-9ce1-08b8316a4558 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.182304007Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"datafile"}
{"msg":"applying SMF actions for region 9f852063-217e-476c-be0f-1cdbe52b6721 running snapshot 5d9df203-ce89-4f7f-9ce1-08b8316a4558 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.183175484Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.183202654Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"uri":"/crucible/0/regions/9f852063-217e-476c-be0f-1cdbe52b6721/snapshots/5d9df203-ce89-4f7f-9ce1-08b8316a4558/run","method":"POST","req_id":"dc1ef83d-ca8b-4c5d-bf1b-bb2ce9ffb462","remote_addr":"[fd00:1122:3344:101::a]:51629","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":26801,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/9f852063-217e-476c-be0f-1cdbe52b6721","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.18759002Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.187694742Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.187797284Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.187814314Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.187864875Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9f852063-217e-476c-be0f-1cdbe52b6721","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.187923516Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9f852063-217e-476c-be0f-1cdbe52b6721","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.188043079Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.18812521Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.188185151Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.188251253Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.197463232Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-9f852063-217e-476c-be0f-1cdbe52b6721-5d9df203-ce89-4f7f-9ce1-08b8316a4558","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.198101955Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-9f852063-217e-476c-be0f-1cdbe52b6721-5d9df203-ce89-4f7f-9ce1-08b8316a4558","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.20143251Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-9f852063-217e-476c-be0f-1cdbe52b6721-5d9df203-ce89-4f7f-9ce1-08b8316a4558","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.213366633Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9f852063-217e-476c-be0f-1cdbe52b6721/.zfs/snapshot/5d9df203-ce89-4f7f-9ce1-08b8316a4558","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.213644188Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.213714699Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.21376953Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.213824902Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.213890813Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.219075814Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.225242214Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"worker"}
{"msg":"region 9f852063-217e-476c-be0f-1cdbe52b6721 running snapshot 5d9df203-ce89-4f7f-9ce1-08b8316a4558 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.225260654Z","hostname":"oxz_crucible_eebe1b4f-0c34-4764-8d37-ddb62b8dc20f","pid":26198,"component":"datafile"}