{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.205243053Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:41024"} {"msg":"region 0aeb84e4-bb1e-404e-899b-93e28689d33f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.264363429Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.265026128Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"uri":"/crucible/0/regions","method":"POST","req_id":"7fc6d0ee-d695-46e7-a500-fd059b5f60f7","remote_addr":"[fd00:1122:3344:101::a]:41024","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":711,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.265043698Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_dbcf1671-d436-42fe-8fb9-48cfc76a8877/crucible/regions/0aeb84e4-bb1e-404e-899b-93e28689d33f","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.274892044Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_dbcf1671-d436-42fe-8fb9-48cfc76a8877/crucible/regions/0aeb84e4-bb1e-404e-899b-93e28689d33f","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.274921644Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"0aeb84e4-bb1e-404e-899b-93e28689d33f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/0aeb84e4-bb1e-404e-899b-93e28689d33f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.351590935Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"region":"0aeb84e4-bb1e-404e-899b-93e28689d33f","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.488389244Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"region":"0aeb84e4-bb1e-404e-899b-93e28689d33f","component":"worker"} {"msg":"region 0aeb84e4-bb1e-404e-899b-93e28689d33f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.488444085Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.488577637Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-0aeb84e4-bb1e-404e-899b-93e28689d33f","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.489034253Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.491025028Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"uri":"/crucible/0/regions","method":"POST","req_id":"9863f617-d91b-41ec-b733-eb17be24f979","remote_addr":"[fd00:1122:3344:101::a]:41024","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":75,"response_code":"200"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0aeb84e4-bb1e-404e-899b-93e28689d33f","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.494918088Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.525142754Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0aeb84e4-bb1e-404e-899b-93e28689d33f","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.532143884Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0aeb84e4-bb1e-404e-899b-93e28689d33f","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.532411437Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.532463808Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.532503918Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.532542759Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.54044151Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.551631713Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.551727294Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.551813665Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.551820545Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.551866546Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-df7816ee-f04b-4015-b8bf-6ad7aeda5340","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.551920487Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.552024538Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.552088059Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.55214306Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.55219402Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.568430958Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:44.568979845Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.689653247Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44920"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.689753738Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"uri":"/crucible/0/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340","method":"GET","req_id":"8e94757f-28f8-4f8e-ab29-38328575ef82","remote_addr":"[fd00:1122:3344:101::a]:44920","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":76,"response_code":"200"} {"msg":"path is \"/data/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.692129959Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"datafile"} {"msg":"dataset is oxp_dbcf1671-d436-42fe-8fb9-48cfc76a8877/crucible/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.700706578Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"datafile"} {"msg":"snapshot is oxp_dbcf1671-d436-42fe-8fb9-48cfc76a8877/crucible/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340@ad8bf836-963b-4a84-865b-9e252cc74dbf","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.709141256Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_dbcf1671-d436-42fe-8fb9-48cfc76a8877/crucible/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340\", \"ad8bf836-963b-4a84-865b-9e252cc74dbf\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.709173497Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.721849309Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"uri":"/crucible/0/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340/snapshots/ad8bf836-963b-4a84-865b-9e252cc74dbf","method":"GET","req_id":"6790d068-efb6-43d7-a270-f4b10aa724af","remote_addr":"[fd00:1122:3344:101::a]:44920","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":31964,"response_code":"200"} {"msg":"path is \"/data/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.722002021Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"datafile"} {"msg":"dataset is oxp_dbcf1671-d436-42fe-8fb9-48cfc76a8877/crucible/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.730107204Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"datafile"} {"msg":"snapshot is oxp_dbcf1671-d436-42fe-8fb9-48cfc76a8877/crucible/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340@ad8bf836-963b-4a84-865b-9e252cc74dbf","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.737473008Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_dbcf1671-d436-42fe-8fb9-48cfc76a8877/crucible/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340\", \"ad8bf836-963b-4a84-865b-9e252cc74dbf\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.737496499Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot df7816ee-f04b-4015-b8bf-6ad7aeda5340-ad8bf836-963b-4a84-865b-9e252cc74dbf state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.74622242Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"datafile"} {"msg":"applying SMF actions for region df7816ee-f04b-4015-b8bf-6ad7aeda5340 running snapshot ad8bf836-963b-4a84-865b-9e252cc74dbf (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.746387722Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.746418793Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"uri":"/crucible/0/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340/snapshots/ad8bf836-963b-4a84-865b-9e252cc74dbf/run","method":"POST","req_id":"6edfd85d-f370-4d71-b28c-d71c18bd1eb4","remote_addr":"[fd00:1122:3344:101::a]:44920","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":24447,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.747996073Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.748082294Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.748163305Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.748170655Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.748210696Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-df7816ee-f04b-4015-b8bf-6ad7aeda5340","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.748256986Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.74855486Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.748613471Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.748663082Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.748710222Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.759696733Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-df7816ee-f04b-4015-b8bf-6ad7aeda5340-ad8bf836-963b-4a84-865b-9e252cc74dbf","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.760337731Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-df7816ee-f04b-4015-b8bf-6ad7aeda5340-ad8bf836-963b-4a84-865b-9e252cc74dbf","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.76731692Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-df7816ee-f04b-4015-b8bf-6ad7aeda5340-ad8bf836-963b-4a84-865b-9e252cc74dbf","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.791714402Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/df7816ee-f04b-4015-b8bf-6ad7aeda5340/.zfs/snapshot/ad8bf836-963b-4a84-865b-9e252cc74dbf","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.792011176Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.792073817Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.792118317Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.792160358Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.792228279Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.799038396Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.804640498Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"worker"} {"msg":"region df7816ee-f04b-4015-b8bf-6ad7aeda5340 running snapshot ad8bf836-963b-4a84-865b-9e252cc74dbf state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-10T17:17:45.804662178Z","hostname":"oxz_crucible_398d8233-2b95-4fab-9283-4d074a18696b","pid":22498,"component":"datafile"}