[ Jan  8 01:15:33 Enabled. ]
[ Jan  8 01:15:33 Rereading configuration. ]
[ Jan  8 01:15:34 Rereading configuration. ]
[ Jan  8 01:15:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6b5fcf41-3599-47ca-b2a4-fdb3b9ab8254/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_6b5fcf41-3599-47ca-b2a4-fdb3b9ab8254/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:15:39.113186282Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:15:39.122116473Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:15:39.122130463Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:15:39.136627961Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:15:39.263974857Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.061797069Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57614"}
{"msg":"region e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.099873028Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.105084891Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.110223401Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"uri":"/crucible/0/regions","method":"POST","req_id":"162e69c9-71a2-4566-9dda-e32b43219c21","remote_addr":"[fd00:1122:3344:101::a]:57614","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":12362,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_6b5fcf41-3599-47ca-b2a4-fdb3b9ab8254/crucible/regions/e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.119219433Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_6b5fcf41-3599-47ca-b2a4-fdb3b9ab8254/crucible/regions/e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.119237193Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.178775761Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"region":"e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2\" \"--data\" \"/data/regions/e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.181049807Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"region":"e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.345742571Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"uri":"/crucible/0/regions","method":"POST","req_id":"1b2f7fc3-9583-4477-a69e-b0f5599fbf15","remote_addr":"[fd00:1122:3344:101::a]:57614","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.484223402Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"region":"e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2","component":"worker"}
{"msg":"region e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.484279703Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.484391915Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.484878773Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.494505144Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.512925244Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.516842586Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e05f82f1-bf44-4b32-b30b-7ce4a67ff2b2","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.517944563Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.518039005Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.518110096Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.518171957Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.523526531Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.543212141Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:13.695627782Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"uri":"/crucible/0/regions","method":"POST","req_id":"bc0e5279-1f85-4282-baad-f01f148ddae5","remote_addr":"[fd00:1122:3344:101::a]:57614","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"region 92bafaf9-88b1-40cd-b227-d454c8afb30d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:18.993233979Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:18.993427552Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"uri":"/crucible/0/regions","method":"POST","req_id":"1e05fd3f-3e05-424a-9b51-1c708323025d","remote_addr":"[fd00:1122:3344:101::a]:57614","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":235,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:18.993444282Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_6b5fcf41-3599-47ca-b2a4-fdb3b9ab8254/crucible/regions/92bafaf9-88b1-40cd-b227-d454c8afb30d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.003675414Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_6b5fcf41-3599-47ca-b2a4-fdb3b9ab8254/crucible/regions/92bafaf9-88b1-40cd-b227-d454c8afb30d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.003731674Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"92bafaf9-88b1-40cd-b227-d454c8afb30d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/92bafaf9-88b1-40cd-b227-d454c8afb30d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.066699455Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"region":"92bafaf9-88b1-40cd-b227-d454c8afb30d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"92bafaf9-88b1-40cd-b227-d454c8afb30d\" \"--data\" \"/data/regions/92bafaf9-88b1-40cd-b227-d454c8afb30d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.066776376Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"region":"92bafaf9-88b1-40cd-b227-d454c8afb30d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.156341847Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"region":"92bafaf9-88b1-40cd-b227-d454c8afb30d","component":"worker"}
{"msg":"region 92bafaf9-88b1-40cd-b227-d454c8afb30d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.156409458Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.15648674Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-92bafaf9-88b1-40cd-b227-d454c8afb30d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.156977657Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-92bafaf9-88b1-40cd-b227-d454c8afb30d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.160719456Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.170304907Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-92bafaf9-88b1-40cd-b227-d454c8afb30d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.173702201Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/92bafaf9-88b1-40cd-b227-d454c8afb30d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.173985635Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.174055026Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.174102547Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.174144428Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.178002848Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.187064131Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:19.214635485Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"uri":"/crucible/0/regions","method":"POST","req_id":"00eb8055-3e58-41e4-8132-1a69ec213ccc","remote_addr":"[fd00:1122:3344:101::a]:57614","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:45.173898362Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"uri":"/crucible/0/regions/92bafaf9-88b1-40cd-b227-d454c8afb30d","method":"GET","req_id":"78d0213f-5bee-46ef-ac63-d1ae6ebf9a49","remote_addr":"[fd00:1122:3344:101::a]:57614","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"path is \"/data/regions/92bafaf9-88b1-40cd-b227-d454c8afb30d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:45.177031872Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"datafile"}
{"msg":"dataset is oxp_6b5fcf41-3599-47ca-b2a4-fdb3b9ab8254/crucible/regions/92bafaf9-88b1-40cd-b227-d454c8afb30d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:45.189103582Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"datafile"}
{"msg":"region 92bafaf9-88b1-40cd-b227-d454c8afb30d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:45.199621358Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:45.199811381Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:45.199864091Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"uri":"/crucible/0/regions/92bafaf9-88b1-40cd-b227-d454c8afb30d","method":"DELETE","req_id":"21833772-e3f2-4b12-b9bb-6f81a6ab1dea","remote_addr":"[fd00:1122:3344:101::a]:57614","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":22866,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-92bafaf9-88b1-40cd-b227-d454c8afb30d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:45.200860397Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:45.203255625Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"uri":"/crucible/0/regions/92bafaf9-88b1-40cd-b227-d454c8afb30d","method":"GET","req_id":"897a39e4-7952-4355-ab6a-c295e9603ddd","remote_addr":"[fd00:1122:3344:101::a]:57614","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:45.206614908Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_6b5fcf41-3599-47ca-b2a4-fdb3b9ab8254/crucible/regions/92bafaf9-88b1-40cd-b227-d454c8afb30d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:45.219358738Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"region":"92bafaf9-88b1-40cd-b227-d454c8afb30d","component":"worker"}
{"msg":"region 92bafaf9-88b1-40cd-b227-d454c8afb30d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:45.292379808Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-08T01:27:45.325285687Z","hostname":"oxz_crucible_219d928b-4029-4404-8163-4303d264f06b","pid":27216,"uri":"/crucible/0/regions/92bafaf9-88b1-40cd-b227-d454c8afb30d","method":"GET","req_id":"55d92844-b359-471a-83c7-1e1e028fe2d8","remote_addr":"[fd00:1122:3344:101::a]:57614","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":63,"response_code":"200"}