[ Jan 17 05:15:26 Enabled. ] [ Jan 17 05:15:26 Rereading configuration. ] [ Jan 17 05:15:27 Rereading configuration. ] [ Jan 17 05:15:30 Executing start method ("/opt/oxide/lib/svc/manifest/crucible/agent.sh"). ] + . /lib/svc/share/smf_include.sh ++ SMF_EXIT_OK=0 ++ SMF_EXIT_NODAEMON=94 ++ SMF_EXIT_ERR_FATAL=95 ++ SMF_EXIT_ERR_CONFIG=96 ++ SMF_EXIT_MON_DEGRADE=97 ++ SMF_EXIT_MON_OFFLINE=98 ++ SMF_EXIT_ERR_NOSMF=99 ++ SMF_EXIT_ERR_PERM=100 ++ svcprop -c -p config/datalink svc:/oxide/crucible/agent:default + DATALINK=oxControlService22 ++ svcprop -c -p config/gateway svc:/oxide/crucible/agent:default + GATEWAY=fd00:1122:3344:101::1 ++ svcprop -c -p config/dataset svc:/oxide/crucible/agent:default + DATASET=oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible ++ svcprop -c -p config/listen_addr svc:/oxide/crucible/agent:default + LISTEN_ADDR=fd00:1122:3344:101::12 ++ svcprop -c -p config/listen_port svc:/oxide/crucible/agent:default + LISTEN_PORT=32345 ++ svcprop -c -p config/portbase svc:/oxide/crucible/agent:default + PORTBASE=19000 ++ svcprop -c -p config/downstairs_prefix svc:/oxide/crucible/agent:default + DOWNSTAIRS_PREFIX=downstairs ++ svcprop -c -p config/snapshot_prefix svc:/oxide/crucible/agent:default + SNAPSHOT_PREFIX=snapshot + [[ oxControlService22 == unknown ]] + [[ fd00:1122:3344:101::1 == unknown ]] + ipadm delete-if oxControlService22 ipadm: Could not delete oxControlService22: Interface does not exist + true + ipadm create-if -t oxControlService22 + ipadm set-ifprop -t -p mtu=9000 -m ipv4 oxControlService22 + ipadm set-ifprop -t -p mtu=9000 -m ipv6 oxControlService22 + ipadm show-addr oxControlService22/ll ipadm: Address object not found + ipadm create-addr -t -T addrconf oxControlService22/ll + ipadm show-addr oxControlService22/omicron6 ipadm: Address object not found + ipadm create-addr -t -T static -a fd00:1122:3344:101::12 oxControlService22/omicron6 + route get -inet6 default -inet6 fd00:1122:3344:101::1 default: not in table + route add -inet6 default -inet6 fd00:1122:3344:101::1 add net default: gateway fd00:1122:3344:101::1 + args=('-D' '/opt/oxide/crucible/bin/crucible-downstairs' '--dataset' "$DATASET" '-l' "[$LISTEN_ADDR]:$LISTEN_PORT" '-P' "$PORTBASE" '-p' "$DOWNSTAIRS_PREFIX" '-s' "$SNAPSHOT_PREFIX") + exec /opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible -l '[fd00:1122:3344:101::12]:32345' -P 19000 -p downstairs -s snapshot note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:15:31.53605982Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963} {"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:15:31.5392861Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:15:31.53929931Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:15:31.907241372Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:19.939142614Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57667"} {"msg":"region 5db0682b-dd42-4282-9948-8aca8be47b49 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:19.978976117Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:19.980607697Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions","method":"POST","req_id":"91e3734e-2f19-4a52-902c-61518fb0eaef","remote_addr":"[fd00:1122:3344:101::a]:57667","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":6849,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:19.980667249Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/5db0682b-dd42-4282-9948-8aca8be47b49","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:19.992801185Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/5db0682b-dd42-4282-9948-8aca8be47b49","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:19.992832285Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"5db0682b-dd42-4282-9948-8aca8be47b49\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/5db0682b-dd42-4282-9948-8aca8be47b49\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.099450204Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"region":"5db0682b-dd42-4282-9948-8aca8be47b49","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.281406868Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"region":"5db0682b-dd42-4282-9948-8aca8be47b49","component":"worker"} {"msg":"region 5db0682b-dd42-4282-9948-8aca8be47b49 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.281452059Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.281742285Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-5db0682b-dd42-4282-9948-8aca8be47b49","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.282226163Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5db0682b-dd42-4282-9948-8aca8be47b49","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.287816098Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.299359073Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5db0682b-dd42-4282-9948-8aca8be47b49","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.306123729Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5db0682b-dd42-4282-9948-8aca8be47b49","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.3066988Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.308950962Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.309002333Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.309050674Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.317150865Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions","method":"POST","req_id":"a8f26afa-a03c-41e6-b404-e73aa7759384","remote_addr":"[fd00:1122:3344:101::a]:57667","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":74,"response_code":"200"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.328948245Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:20.339728866Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.055824302Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:38447"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.055916594Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions/5db0682b-dd42-4282-9948-8aca8be47b49","method":"GET","req_id":"9658918b-627e-4b0a-b5e8-53fedb782d68","remote_addr":"[fd00:1122:3344:101::a]:38447","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":69,"response_code":"200"} {"msg":"path is \"/data/regions/5db0682b-dd42-4282-9948-8aca8be47b49\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.056097208Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/5db0682b-dd42-4282-9948-8aca8be47b49","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.065744728Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"region 5db0682b-dd42-4282-9948-8aca8be47b49 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.075427598Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.075672253Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.075723994Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions/5db0682b-dd42-4282-9948-8aca8be47b49","method":"DELETE","req_id":"f3eb8150-3c5d-4816-8a41-c48dff91dd29","remote_addr":"[fd00:1122:3344:101::a]:38447","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":19665,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.075868436Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions/5db0682b-dd42-4282-9948-8aca8be47b49","method":"GET","req_id":"a070ce59-b914-40e4-893d-e1afc8ccd05f","remote_addr":"[fd00:1122:3344:101::a]:38447","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":21,"response_code":"200"} {"msg":"disabling downstairs instance: downstairs-5db0682b-dd42-4282-9948-8aca8be47b49 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.076852865Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.085413504Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/5db0682b-dd42-4282-9948-8aca8be47b49\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.096284497Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"region":"5db0682b-dd42-4282-9948-8aca8be47b49","component":"worker"} {"msg":"region 5db0682b-dd42-4282-9948-8aca8be47b49 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.163703375Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:21.221133636Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions/5db0682b-dd42-4282-9948-8aca8be47b49","method":"GET","req_id":"51ee171c-ba50-42a0-a180-3871fc7766d9","remote_addr":"[fd00:1122:3344:101::a]:38447","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":50,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:22.8243609Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53417"} {"msg":"region 70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:22.824462001Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:22.824639275Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions","method":"POST","req_id":"09da0029-adff-4eaf-8e73-552019a260d7","remote_addr":"[fd00:1122:3344:101::a]:53417","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":219,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:22.824696886Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:22.833575231Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:22.833606972Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:22.90481335Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"region":"70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:22.966883948Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions","method":"POST","req_id":"c335989e-85a8-4678-9d52-013d5423a380","remote_addr":"[fd00:1122:3344:101::a]:53417","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":88,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.013397576Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"region":"70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0","component":"worker"} {"msg":"region 70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.013423636Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.013791793Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.01522185Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.022048227Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.04474691Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.058159801Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.058495177Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.058540478Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.058588389Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.058627809Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.069009783Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.080913955Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:16:23.559160196Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions","method":"POST","req_id":"f970ec76-f873-4e00-b6fa-92abc9cad1ee","remote_addr":"[fd00:1122:3344:101::a]:53417","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":52,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.296970341Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37115"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.297151055Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0","method":"GET","req_id":"8208d9af-962a-470e-8dfb-64873f2eae3d","remote_addr":"[fd00:1122:3344:101::a]:37115","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":59,"response_code":"200"} {"msg":"path is \"/data/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.297207765Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.305607412Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"snapshot is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0@f9283a43-16a4-41fc-928a-3b036fb10d47","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.313300486Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0\", \"f9283a43-16a4-41fc-928a-3b036fb10d47\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.313342946Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.325967582Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0/snapshots/f9283a43-16a4-41fc-928a-3b036fb10d47","method":"GET","req_id":"17ed557e-0f58-480a-8ac8-b6861bda0711","remote_addr":"[fd00:1122:3344:101::a]:37115","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":28764,"response_code":"200"} {"msg":"path is \"/data/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.326141135Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.335308406Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"snapshot is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0@f9283a43-16a4-41fc-928a-3b036fb10d47","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.343760134Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0\", \"f9283a43-16a4-41fc-928a-3b036fb10d47\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.343787374Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0-f9283a43-16a4-41fc-928a-3b036fb10d47 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.352901074Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"applying SMF actions for region 70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0 running snapshot f9283a43-16a4-41fc-928a-3b036fb10d47 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.353532206Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.353569057Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0/snapshots/f9283a43-16a4-41fc-928a-3b036fb10d47/run","method":"POST","req_id":"eaf7c1eb-6d2a-4eb0-ba42-c204a66ff4c2","remote_addr":"[fd00:1122:3344:101::a]:37115","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":27466,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0-f9283a43-16a4-41fc-928a-3b036fb10d47","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.362082686Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0-f9283a43-16a4-41fc-928a-3b036fb10d47","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.36551349Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0-f9283a43-16a4-41fc-928a-3b036fb10d47","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.381359455Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0/.zfs/snapshot/f9283a43-16a4-41fc-928a-3b036fb10d47","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.38162809Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.381684541Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.381729702Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.381770393Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.381822524Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.385214827Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.389572208Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"region 70129f8d-ee3b-443e-b4ff-5fe0eb6dfce0 running snapshot f9283a43-16a4-41fc-928a-3b036fb10d47 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:55.389588659Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.608680269Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54609"} {"msg":"region bb697436-c903-4a2c-a1fb-0df0c2a32c54 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.609395303Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.609575806Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions","method":"POST","req_id":"4900f503-d825-421d-99c9-5e65889ac1bf","remote_addr":"[fd00:1122:3344:101::a]:54609","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":847,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.609614417Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/bb697436-c903-4a2c-a1fb-0df0c2a32c54","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.619615433Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/bb697436-c903-4a2c-a1fb-0df0c2a32c54","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.619659254Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"bb697436-c903-4a2c-a1fb-0df0c2a32c54\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/bb697436-c903-4a2c-a1fb-0df0c2a32c54\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.692397821Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"region":"bb697436-c903-4a2c-a1fb-0df0c2a32c54","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.739336447Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions","method":"POST","req_id":"5672e37b-5ae6-46fa-8dfc-fe4d82391055","remote_addr":"[fd00:1122:3344:101::a]:54609","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":65,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.759384941Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"region":"bb697436-c903-4a2c-a1fb-0df0c2a32c54","component":"worker"} {"msg":"region bb697436-c903-4a2c-a1fb-0df0c2a32c54 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.759422141Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.759543443Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-bb697436-c903-4a2c-a1fb-0df0c2a32c54","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.761267756Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bb697436-c903-4a2c-a1fb-0df0c2a32c54","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.765147478Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.77333081Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bb697436-c903-4a2c-a1fb-0df0c2a32c54","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.776461439Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bb697436-c903-4a2c-a1fb-0df0c2a32c54","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.776734374Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.776782565Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.776826476Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.776870237Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.780194809Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:57.785901555Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:21:58.146138294Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions","method":"POST","req_id":"498c2188-ba89-41d7-9443-fa4d88d39f5f","remote_addr":"[fd00:1122:3344:101::a]:54609","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.698823521Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58233"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.698894302Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions/bb697436-c903-4a2c-a1fb-0df0c2a32c54","method":"GET","req_id":"3d501ef2-34c2-46af-a533-b8656c44190f","remote_addr":"[fd00:1122:3344:101::a]:58233","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":58,"response_code":"200"} {"msg":"path is \"/data/regions/bb697436-c903-4a2c-a1fb-0df0c2a32c54\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.699069236Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/bb697436-c903-4a2c-a1fb-0df0c2a32c54","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.710329896Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"region bb697436-c903-4a2c-a1fb-0df0c2a32c54 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.71911479Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.719330244Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.719370204Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions/bb697436-c903-4a2c-a1fb-0df0c2a32c54","method":"DELETE","req_id":"0086095f-d68b-4595-8ce1-5f913c4ebb69","remote_addr":"[fd00:1122:3344:101::a]:58233","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20336,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.719564778Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions/bb697436-c903-4a2c-a1fb-0df0c2a32c54","method":"GET","req_id":"15258337-a40f-42d9-bc00-8dda661fd94b","remote_addr":"[fd00:1122:3344:101::a]:58233","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":27,"response_code":"200"} {"msg":"disabling downstairs instance: downstairs-bb697436-c903-4a2c-a1fb-0df0c2a32c54 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.721097867Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.727047748Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/bb697436-c903-4a2c-a1fb-0df0c2a32c54\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.736906082Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"region":"bb697436-c903-4a2c-a1fb-0df0c2a32c54","component":"worker"} {"msg":"region bb697436-c903-4a2c-a1fb-0df0c2a32c54 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.784858586Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-17T05:22:28.800908995Z","hostname":"oxz_crucible_b5f0a9a2-efe1-4743-98be-518f62d32488","pid":26963,"uri":"/crucible/0/regions/bb697436-c903-4a2c-a1fb-0df0c2a32c54","method":"GET","req_id":"a3c07eed-55e1-4c06-8c08-f497010c70fc","remote_addr":"[fd00:1122:3344:101::a]:58233","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":1045,"response_code":"200"}