[ Nov 26 23:31:55 Enabled. ]
[ Nov 26 23:31:55 Rereading configuration. ]
[ Nov 26 23:31:57 Rereading configuration. ]
[ Nov 26 23:32:00 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_5d28905a-1799-458a-b4a2-d57daa3e23d5/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_5d28905a-1799-458a-b4a2-d57daa3e23d5/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:00.540533877Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:00.544373103Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:00.544389313Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:00.565778773Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:00.63724873Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:32.762938573Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:40694"}
{"msg":"region 4dbcb388-f882-434c-97f7-3ec9f1f39219 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:32.807007025Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:32.811929241Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:32.816858127Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"uri":"/crucible/0/regions","method":"POST","req_id":"9e25c7ee-84cf-431c-be93-dab1088990af","remote_addr":"[fd00:1122:3344:101::a]:40694","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":16766,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_5d28905a-1799-458a-b4a2-d57daa3e23d5/crucible/regions/4dbcb388-f882-434c-97f7-3ec9f1f39219","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:32.824509218Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_5d28905a-1799-458a-b4a2-d57daa3e23d5/crucible/regions/4dbcb388-f882-434c-97f7-3ec9f1f39219","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:32.824530198Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4dbcb388-f882-434c-97f7-3ec9f1f39219\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4dbcb388-f882-434c-97f7-3ec9f1f39219\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:32.873088381Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"region":"4dbcb388-f882-434c-97f7-3ec9f1f39219","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4dbcb388-f882-434c-97f7-3ec9f1f39219\" \"--data\" \"/data/regions/4dbcb388-f882-434c-97f7-3ec9f1f39219\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:32.875272671Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"region":"4dbcb388-f882-434c-97f7-3ec9f1f39219","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.032526979Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"uri":"/crucible/0/regions","method":"POST","req_id":"1cc654ca-5f74-42aa-98fa-8fa33325425f","remote_addr":"[fd00:1122:3344:101::a]:40694","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.167538319Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"region":"4dbcb388-f882-434c-97f7-3ec9f1f39219","component":"worker"}
{"msg":"region 4dbcb388-f882-434c-97f7-3ec9f1f39219 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.16758099Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.167705391Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4dbcb388-f882-434c-97f7-3ec9f1f39219","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.168089594Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4dbcb388-f882-434c-97f7-3ec9f1f39219","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.178736643Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.196682271Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4dbcb388-f882-434c-97f7-3ec9f1f39219","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.200525947Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4dbcb388-f882-434c-97f7-3ec9f1f39219","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.201252774Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.201315344Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.201357345Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.201395855Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.206365421Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.211345778Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:33.329737333Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"uri":"/crucible/0/regions","method":"POST","req_id":"566dec7c-d7cd-4d5f-89cd-d389643159eb","remote_addr":"[fd00:1122:3344:101::a]:40694","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"region b779ddca-4f1e-4e90-8649-273ef8607854 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.595332019Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.595505501Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"uri":"/crucible/0/regions","method":"POST","req_id":"41476781-ba87-4663-929e-1c16afdbb717","remote_addr":"[fd00:1122:3344:101::a]:40694","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":225,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.595524381Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_5d28905a-1799-458a-b4a2-d57daa3e23d5/crucible/regions/b779ddca-4f1e-4e90-8649-273ef8607854","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.60503068Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_5d28905a-1799-458a-b4a2-d57daa3e23d5/crucible/regions/b779ddca-4f1e-4e90-8649-273ef8607854","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.60507048Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b779ddca-4f1e-4e90-8649-273ef8607854\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b779ddca-4f1e-4e90-8649-273ef8607854\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.657636471Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"region":"b779ddca-4f1e-4e90-8649-273ef8607854","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b779ddca-4f1e-4e90-8649-273ef8607854\" \"--data\" \"/data/regions/b779ddca-4f1e-4e90-8649-273ef8607854\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.657698421Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"region":"b779ddca-4f1e-4e90-8649-273ef8607854","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.74118622Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"region":"b779ddca-4f1e-4e90-8649-273ef8607854","component":"worker"}
{"msg":"region b779ddca-4f1e-4e90-8649-273ef8607854 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.741225101Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.741313561Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b779ddca-4f1e-4e90-8649-273ef8607854","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.745232868Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b779ddca-4f1e-4e90-8649-273ef8607854","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.748098345Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.756424642Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b779ddca-4f1e-4e90-8649-273ef8607854","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.760789243Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b779ddca-4f1e-4e90-8649-273ef8607854","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.761038606Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.761100156Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.761143987Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.761183917Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.765032813Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.770622195Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:43:38.842396395Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"uri":"/crucible/0/regions","method":"POST","req_id":"e370c74d-490f-4065-9713-4e56bbddb86d","remote_addr":"[fd00:1122:3344:101::a]:40694","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.486569539Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"uri":"/crucible/0/regions/b779ddca-4f1e-4e90-8649-273ef8607854","method":"GET","req_id":"19492a19-759b-49b6-98bc-3f4d6411de21","remote_addr":"[fd00:1122:3344:101::a]:40694","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"path is \"/data/regions/b779ddca-4f1e-4e90-8649-273ef8607854\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.490505436Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"datafile"}
{"msg":"dataset is oxp_5d28905a-1799-458a-b4a2-d57daa3e23d5/crucible/regions/b779ddca-4f1e-4e90-8649-273ef8607854","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.502060784Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"datafile"}
{"msg":"region b779ddca-4f1e-4e90-8649-273ef8607854 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.511567023Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.511746034Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.511811815Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"uri":"/crucible/0/regions/b779ddca-4f1e-4e90-8649-273ef8607854","method":"DELETE","req_id":"b1f305c4-52f1-4303-8698-5fa21eb671e4","remote_addr":"[fd00:1122:3344:101::a]:40694","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21335,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-b779ddca-4f1e-4e90-8649-273ef8607854 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.512993956Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.515908633Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"uri":"/crucible/0/regions/b779ddca-4f1e-4e90-8649-273ef8607854","method":"GET","req_id":"099e11ba-37ba-4359-9521-de35c04e9ef3","remote_addr":"[fd00:1122:3344:101::a]:40694","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.532982892Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_5d28905a-1799-458a-b4a2-d57daa3e23d5/crucible/regions/b779ddca-4f1e-4e90-8649-273ef8607854\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.541933766Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"region":"b779ddca-4f1e-4e90-8649-273ef8607854","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.619166327Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"uri":"/crucible/0/regions/b779ddca-4f1e-4e90-8649-273ef8607854","method":"GET","req_id":"349c0b8b-743b-4738-b4f1-6a8d9e134e35","remote_addr":"[fd00:1122:3344:101::a]:40694","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":637,"response_code":"200"}
{"msg":"region b779ddca-4f1e-4e90-8649-273ef8607854 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.713263525Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:44:06.774251774Z","hostname":"oxz_crucible_963329c3-3300-417e-b313-f70de5094b60","pid":23127,"uri":"/crucible/0/regions/b779ddca-4f1e-4e90-8649-273ef8607854","method":"GET","req_id":"553c4d15-f55e-4659-8a15-87d29b6ef95b","remote_addr":"[fd00:1122:3344:101::a]:40694","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":36,"response_code":"200"}