[ Mar 28 08:13:24 Enabled. ]
[ Mar 28 08:13:24 Rereading configuration. ]
[ Mar 28 08:13:25 Rereading configuration. ]
[ Mar 28 08:13:27 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:27.60098444Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:27.601282985Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:27.601292225Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:27.677645779Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:47.718512987Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:42076"}
{"msg":"region b0a468ca-3c88-42da-9e59-cf822cc30766 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:47.780326739Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:47.781155244Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:47.781176094Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"uri":"/crucible/0/regions","method":"POST","req_id":"9f9737c2-2a21-477c-93e6-1ebba660e73d","remote_addr":"[fd00:1122:3344:101::c]:42076","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":1314,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/b0a468ca-3c88-42da-9e59-cf822cc30766","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:47.793364096Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/b0a468ca-3c88-42da-9e59-cf822cc30766","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:47.793395596Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b0a468ca-3c88-42da-9e59-cf822cc30766\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/b0a468ca-3c88-42da-9e59-cf822cc30766\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:47.880630679Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"region":"b0a468ca-3c88-42da-9e59-cf822cc30766","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:47.952809282Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"uri":"/crucible/0/regions","method":"POST","req_id":"f3c93e19-a86a-4720-a3f9-faf5d0d0fddf","remote_addr":"[fd00:1122:3344:101::c]:42076","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.007254766Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"region":"b0a468ca-3c88-42da-9e59-cf822cc30766","component":"worker"}
{"msg":"region b0a468ca-3c88-42da-9e59-cf822cc30766 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.007287367Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.007430179Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b0a468ca-3c88-42da-9e59-cf822cc30766","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.007856337Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b0a468ca-3c88-42da-9e59-cf822cc30766","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.010595134Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.017751558Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b0a468ca-3c88-42da-9e59-cf822cc30766","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.020764841Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b0a468ca-3c88-42da-9e59-cf822cc30766","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.021500209Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.021571826Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.021616396Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.021655818Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.02533892Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.029142666Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.534194898Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"uri":"/crucible/0/regions","method":"POST","req_id":"1a52261a-1b69-4883-abc8-3920308873fd","remote_addr":"[fd00:1122:3344:101::c]:42076","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.824998794Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:50127"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.825890629Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"uri":"/crucible/0/regions/b0a468ca-3c88-42da-9e59-cf822cc30766","method":"GET","req_id":"fc11b437-cd41-4675-96c6-510b6aab3af2","remote_addr":"[fd00:1122:3344:101::c]:50127","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":823,"response_code":"200"}
{"msg":"path is \"/data/regions/b0a468ca-3c88-42da-9e59-cf822cc30766\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.826151544Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"datafile"}
{"msg":"dataset is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/b0a468ca-3c88-42da-9e59-cf822cc30766","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.835119479Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"datafile"}
{"msg":"region b0a468ca-3c88-42da-9e59-cf822cc30766 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.842807323Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.842979846Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.843047487Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"uri":"/crucible/0/regions/b0a468ca-3c88-42da-9e59-cf822cc30766","method":"DELETE","req_id":"8b86e7e3-a002-4107-8ef6-360b9bda46b7","remote_addr":"[fd00:1122:3344:101::c]:50127","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":16941,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.843193019Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"uri":"/crucible/0/regions/b0a468ca-3c88-42da-9e59-cf822cc30766","method":"GET","req_id":"b34cfd3e-bcf1-41e0-aaa1-98e2b244e266","remote_addr":"[fd00:1122:3344:101::c]:50127","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-b0a468ca-3c88-42da-9e59-cf822cc30766 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.84440296Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.847961512Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/b0a468ca-3c88-42da-9e59-cf822cc30766\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.856771255Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"region":"b0a468ca-3c88-42da-9e59-cf822cc30766","component":"worker"}
{"msg":"region b0a468ca-3c88-42da-9e59-cf822cc30766 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.912463741Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:13:48.927704266Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"uri":"/crucible/0/regions/b0a468ca-3c88-42da-9e59-cf822cc30766","method":"GET","req_id":"ae6d3df0-a42f-447e-bdc3-c898c1b0a8ea","remote_addr":"[fd00:1122:3344:101::c]:50127","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":111,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.196509731Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47027"}
{"msg":"region 1be757ef-20ca-4926-ae36-2a4c2161a870 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.197177753Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.197348416Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"uri":"/crucible/0/regions","method":"POST","req_id":"1ae794a6-3d35-4960-a922-3d1ccac995cd","remote_addr":"[fd00:1122:3344:101::a]:47027","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":802,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.197366897Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/1be757ef-20ca-4926-ae36-2a4c2161a870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.206621807Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/1be757ef-20ca-4926-ae36-2a4c2161a870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.206653658Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1be757ef-20ca-4926-ae36-2a4c2161a870\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/1be757ef-20ca-4926-ae36-2a4c2161a870\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.267196848Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"region":"1be757ef-20ca-4926-ae36-2a4c2161a870","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.334723049Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"region":"1be757ef-20ca-4926-ae36-2a4c2161a870","component":"worker"}
{"msg":"region 1be757ef-20ca-4926-ae36-2a4c2161a870 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.334776501Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.334874602Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1be757ef-20ca-4926-ae36-2a4c2161a870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.336266936Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1be757ef-20ca-4926-ae36-2a4c2161a870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.338939923Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.345969185Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1be757ef-20ca-4926-ae36-2a4c2161a870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.349235861Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1be757ef-20ca-4926-ae36-2a4c2161a870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.349536737Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.349598428Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.349641688Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.349683679Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.353574187Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.35836639Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:14:36.522814473Z","hostname":"oxz_crucible_12b2f995-52dc-48a9-9f09-ebd02803e9bb","pid":23074,"uri":"/crucible/0/regions","method":"POST","req_id":"d761bee3-c256-43bd-8e1e-69ae814a89ed","remote_addr":"[fd00:1122:3344:101::a]:47027","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":75,"response_code":"200"}