[ Nov 28 01:34:52 Enabled. ]
[ Nov 28 01:34:52 Rereading configuration. ]
[ Nov 28 01:34:56 Rereading configuration. ]
[ Nov 28 01:35:01 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8164ba35-0eff-436a-9b6b-80912419e510/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_8164ba35-0eff-436a-9b6b-80912419e510/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:35:01.103509936Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:35:01.106848362Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:35:01.106858052Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:35:01.121697287Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:35:01.226958928Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:05.940626568Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52821"}
{"msg":"region 118a3aa2-182e-4bfb-b960-23ae3c3fac59 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:05.965970947Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:05.970040724Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:05.974921615Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"uri":"/crucible/0/regions","method":"POST","req_id":"05c53f22-0526-4ad3-b8d3-b7a510291db6","remote_addr":"[fd00:1122:3344:101::a]:52821","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":13470,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_8164ba35-0eff-436a-9b6b-80912419e510/crucible/regions/118a3aa2-182e-4bfb-b960-23ae3c3fac59","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:05.981206779Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_8164ba35-0eff-436a-9b6b-80912419e510/crucible/regions/118a3aa2-182e-4bfb-b960-23ae3c3fac59","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:05.981228539Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"118a3aa2-182e-4bfb-b960-23ae3c3fac59\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/118a3aa2-182e-4bfb-b960-23ae3c3fac59\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.055697721Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"region":"118a3aa2-182e-4bfb-b960-23ae3c3fac59","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"118a3aa2-182e-4bfb-b960-23ae3c3fac59\" \"--data\" \"/data/regions/118a3aa2-182e-4bfb-b960-23ae3c3fac59\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.055768122Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"region":"118a3aa2-182e-4bfb-b960-23ae3c3fac59","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.156376896Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"uri":"/crucible/0/regions","method":"POST","req_id":"06b93bcf-dbc1-4e6f-afb6-8edd6cc102e0","remote_addr":"[fd00:1122:3344:101::a]:52821","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.181795716Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"region":"118a3aa2-182e-4bfb-b960-23ae3c3fac59","component":"worker"}
{"msg":"region 118a3aa2-182e-4bfb-b960-23ae3c3fac59 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.181821857Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.181956319Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-118a3aa2-182e-4bfb-b960-23ae3c3fac59","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.182337236Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-118a3aa2-182e-4bfb-b960-23ae3c3fac59","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.187601313Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.198211878Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-118a3aa2-182e-4bfb-b960-23ae3c3fac59","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.201175547Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/118a3aa2-182e-4bfb-b960-23ae3c3fac59","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.201747557Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.201806478Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.201850198Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.201889589Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.205466838Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.213697064Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:06.864666931Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"uri":"/crucible/0/regions","method":"POST","req_id":"0aa479da-285d-438b-88bd-ba56c8680a9f","remote_addr":"[fd00:1122:3344:101::a]:52821","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:07.491849769Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"uri":"/crucible/0/regions/118a3aa2-182e-4bfb-b960-23ae3c3fac59","method":"GET","req_id":"c7137a49-a33a-45b5-a735-8ec366b82621","remote_addr":"[fd00:1122:3344:101::a]:52821","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"path is \"/data/regions/118a3aa2-182e-4bfb-b960-23ae3c3fac59\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:07.494664931Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"datafile"}
{"msg":"dataset is oxp_8164ba35-0eff-436a-9b6b-80912419e510/crucible/regions/118a3aa2-182e-4bfb-b960-23ae3c3fac59","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:07.50604769Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"datafile"}
{"msg":"region 118a3aa2-182e-4bfb-b960-23ae3c3fac59 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:07.516626205Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:07.516774187Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:07.516835688Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"uri":"/crucible/0/regions/118a3aa2-182e-4bfb-b960-23ae3c3fac59","method":"DELETE","req_id":"a190e5ae-0a00-478e-b703-d353ecaa881d","remote_addr":"[fd00:1122:3344:101::a]:52821","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":22195,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:07.519814237Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"uri":"/crucible/0/regions/118a3aa2-182e-4bfb-b960-23ae3c3fac59","method":"GET","req_id":"28457870-494e-4030-8922-c4aecdc2bad0","remote_addr":"[fd00:1122:3344:101::a]:52821","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-118a3aa2-182e-4bfb-b960-23ae3c3fac59 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:07.519862028Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:07.52480415Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8164ba35-0eff-436a-9b6b-80912419e510/crucible/regions/118a3aa2-182e-4bfb-b960-23ae3c3fac59\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:07.53570565Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"region":"118a3aa2-182e-4bfb-b960-23ae3c3fac59","component":"worker"}
{"msg":"region 118a3aa2-182e-4bfb-b960-23ae3c3fac59 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:07.599831221Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-28T01:36:07.626562453Z","hostname":"oxz_crucible_4ef451f1-362e-4b8f-b326-cebe912a93f2","pid":22863,"uri":"/crucible/0/regions/118a3aa2-182e-4bfb-b960-23ae3c3fac59","method":"GET","req_id":"0af70d17-a4ef-4821-92db-9298ec3e1b19","remote_addr":"[fd00:1122:3344:101::a]:52821","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":70,"response_code":"200"}