[ Oct 30 17:49:03 Enabled. ]
[ Oct 30 17:49:03 Rereading configuration. ]
[ Oct 30 17:49:04 Rereading configuration. ]
[ Oct 30 17:49:06 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:06.383363501Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:06.385154773Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:06.385168904Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:06.413536755Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:49:06.549154564Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.353969787Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42081"}
{"msg":"region 2edeea3b-d55c-47eb-9a7f-5f4001143089 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.38439734Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.389108125Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions","method":"POST","req_id":"a37d8c6d-3369-45bf-81f2-2ea51ab8f0e4","remote_addr":"[fd00:1122:3344:101::a]:42081","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":12543,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.391780318Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.400522454Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.400559555Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2edeea3b-d55c-47eb-9a7f-5f4001143089\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.449696442Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"region":"2edeea3b-d55c-47eb-9a7f-5f4001143089","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2edeea3b-d55c-47eb-9a7f-5f4001143089\" \"--data\" \"/data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.453547906Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"region":"2edeea3b-d55c-47eb-9a7f-5f4001143089","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.639201033Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions","method":"POST","req_id":"7b11b627-cfa6-4cd2-8077-38804e56af89","remote_addr":"[fd00:1122:3344:101::a]:42081","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.659346668Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"region":"2edeea3b-d55c-47eb-9a7f-5f4001143089","component":"worker"}
{"msg":"region 2edeea3b-d55c-47eb-9a7f-5f4001143089 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.65941864Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.659674409Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.661974659Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.667462831Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.68975058Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.694689702Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.702306558Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.702383461Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.702437053Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.702471214Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.710813746Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.718239605Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:50:12.954644746Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions","method":"POST","req_id":"bb36b987-cf58-4b1a-a32d-bd5a747a649b","remote_addr":"[fd00:1122:3344:101::a]:42081","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.913148998Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63657"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.91634714Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","method":"GET","req_id":"1ee20717-e862-4711-a1b6-9677a5f79cf6","remote_addr":"[fd00:1122:3344:101::a]:63657","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":3155,"response_code":"200"}
{"msg":"path is \"/data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.919914994Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"dataset is oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.929435177Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"snapshot is oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089@8c32b30e-81b1-4b61-a164-c15be5459d59","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.937847981Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089\", \"8c32b30e-81b1-4b61-a164-c15be5459d59\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.937879482Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.9515684Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089/snapshots/8c32b30e-81b1-4b61-a164-c15be5459d59","method":"GET","req_id":"2862a473-8fe8-45f0-b67a-b916a7d521cd","remote_addr":"[fd00:1122:3344:101::a]:63657","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":31652,"response_code":"200"}
{"msg":"path is \"/data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.954543084Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"dataset is oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.96273079Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"snapshot is oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089@8c32b30e-81b1-4b61-a164-c15be5459d59","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.971005789Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089\", \"8c32b30e-81b1-4b61-a164-c15be5459d59\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.9710378Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 2edeea3b-d55c-47eb-9a7f-5f4001143089-8c32b30e-81b1-4b61-a164-c15be5459d59 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.979485426Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"applying SMF actions for region 2edeea3b-d55c-47eb-9a7f-5f4001143089 running snapshot 8c32b30e-81b1-4b61-a164-c15be5459d59 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.980161699Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.9801831Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089/snapshots/8c32b30e-81b1-4b61-a164-c15be5459d59/run","method":"POST","req_id":"d7b8371a-b44e-49a9-b7fb-e38d087e1809","remote_addr":"[fd00:1122:3344:101::a]:63657","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25674,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.988796781Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.988903655Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.989003418Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.989011699Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.98906515Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.989123212Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.989236266Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.989310179Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.989365621Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.989415413Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.999184304Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-2edeea3b-d55c-47eb-9a7f-5f4001143089-8c32b30e-81b1-4b61-a164-c15be5459d59","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:33.999669931Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-2edeea3b-d55c-47eb-9a7f-5f4001143089-8c32b30e-81b1-4b61-a164-c15be5459d59","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.005338439Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-2edeea3b-d55c-47eb-9a7f-5f4001143089-8c32b30e-81b1-4b61-a164-c15be5459d59","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.022923773Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089/.zfs/snapshot/8c32b30e-81b1-4b61-a164-c15be5459d59","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.023228454Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.023283776Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.023326967Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.023363659Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.023412641Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.030142616Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.037778322Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"region 2edeea3b-d55c-47eb-9a7f-5f4001143089 running snapshot 8c32b30e-81b1-4b61-a164-c15be5459d59 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.037793183Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"path is \"/data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.229262122Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"dataset is oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.23807469Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"snapshot is oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089@8c32b30e-81b1-4b61-a164-c15be5459d59","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.246444503Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089\", \"8c32b30e-81b1-4b61-a164-c15be5459d59\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.246495334Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:34.254292587Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089/snapshots/8c32b30e-81b1-4b61-a164-c15be5459d59/run","method":"POST","req_id":"00108a03-1d99-43f4-be29-764fe881f621","remote_addr":"[fd00:1122:3344:101::a]:63657","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25054,"response_code":"200"}
{"msg":"region aec8cb83-e265-4245-8fc4-83d9a8ad4ef3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.662359158Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.662554095Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions","method":"POST","req_id":"a19452b2-111f-48a2-9fd2-2da8d824f589","remote_addr":"[fd00:1122:3344:101::a]:63657","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":275,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.662583646Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.673097634Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.673128835Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"aec8cb83-e265-4245-8fc4-83d9a8ad4ef3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/aec8cb83-e265-4245-8fc4-83d9a8ad4ef3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.752906122Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"region":"aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"aec8cb83-e265-4245-8fc4-83d9a8ad4ef3\" \"--data\" \"/data/regions/aec8cb83-e265-4245-8fc4-83d9a8ad4ef3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.754787578Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"region":"aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.858637787Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"region":"aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","component":"worker"}
{"msg":"region aec8cb83-e265-4245-8fc4-83d9a8ad4ef3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.858677758Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.858793022Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.859398493Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.859479446Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.859558149Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.859569259Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.859607151Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.859657672Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.860138039Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.860215672Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.860264474Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.860316955Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.867954852Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.86846498Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.878021034Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.890662086Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.903647789Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.903853697Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.903912889Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.90395609Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.903999092Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.914407925Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:38.921296266Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:53:39.016899037Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions","method":"POST","req_id":"10b34a64-e2c1-4032-8029-bc111b65f2ae","remote_addr":"[fd00:1122:3344:101::a]:63657","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.620119832Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions/aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","method":"GET","req_id":"905e20c6-c17c-4752-9c20-7837e39a628d","remote_addr":"[fd00:1122:3344:101::a]:63657","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":47,"response_code":"200"}
{"msg":"path is \"/data/regions/aec8cb83-e265-4245-8fc4-83d9a8ad4ef3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.623294963Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"dataset is oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.635630194Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"region aec8cb83-e265-4245-8fc4-83d9a8ad4ef3 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.646414151Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.646620098Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-aec8cb83-e265-4245-8fc4-83d9a8ad4ef3 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.647930944Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.649081894Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions/aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","method":"DELETE","req_id":"ff8abe86-e2b3-4694-b9aa-17f729b44c60","remote_addr":"[fd00:1122:3344:101::a]:63657","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25865,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.652383629Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions/aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","method":"GET","req_id":"eaaa1df4-be20-4117-9605-72c20a23b73d","remote_addr":"[fd00:1122:3344:101::a]:63657","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.6524088Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.652488763Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.652581356Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.652601957Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.652632288Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.65268118Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2edeea3b-d55c-47eb-9a7f-5f4001143089","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.653042623Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.653124025Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.653180687Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.65324221Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.657167397Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.658624987Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_c84ed810-b8fd-4283-8511-603d43b629a7/crucible/regions/aec8cb83-e265-4245-8fc4-83d9a8ad4ef3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.670192102Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"region":"aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","component":"worker"}
{"msg":"region aec8cb83-e265-4245-8fc4-83d9a8ad4ef3 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.735339638Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T17:54:03.799268222Z","hostname":"oxz_crucible_530c539e-4272-4675-882c-e4d1d574c146","pid":21437,"uri":"/crucible/0/regions/aec8cb83-e265-4245-8fc4-83d9a8ad4ef3","method":"GET","req_id":"47a159ad-d94f-48b9-8446-df0ba8f50f2b","remote_addr":"[fd00:1122:3344:101::a]:63657","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":80,"response_code":"200"}