%PDF-1.5 % 1 0 obj<> endobj 2 0 obj<> endobj 3 0 obj<> endobj 5 0 obj null endobj 6 0 obj<> endobj 7 0 obj[/Indexed/DeviceRGB 15 9 0 R] endobj 8 0 obj 48 endobj 9 0 obj<>stream ͖W΅+r< endstream endobj 10 0 obj 151 endobj 11 0 obj<>stream Hd D!+X_x;1L ,1Xfw\狲q8V_G!LzcC3B"46Wڊ+l%yՓ'U^޵9Oi2;\_nunV2aK endstream endobj 12 0 obj(http://www.managementhelp.org/ap-search-go-btn.gif) endobj 13 0 obj(ߚޱb3) endobj 14 0 obj<> endobj 15 0 obj<> endobj 16 0 obj<> endobj 17 0 obj<> endobj 18 0 obj<> endobj 19 0 obj[/Indexed/DeviceRGB 127 21 0 R] endobj 20 0 obj 384 endobj 21 0 obj<>stream NNN@@@Y|N/8```ppp j000;$)))s籱PPP rrr`:rE)N_#;;;aۃr*4>&|L]9N0م1 {-h&Ui&|MmC/  -78r) endstream endobj 22 0 obj 1081 endobj 23 0 obj<>stream Hr"9 %}74@s$drgf~GZnXHz*L*n>YM> nSTQ3YQlq(mƠAzgT!ljwz_k^(]=<>*,\ۯ|eagj*jt581 }375[q +i2Θ P%)uCN1ĺ/(u)fM1jH9n һMyHYS\ѓ7\`i3]8#4r)3'&wՕ/tG7W#eQ?c)t;~L b-f 6P{sLoK_` 2}$C/xmaS%6AM=} 2U7%T6|T>tt MYk{;=qGjS^6LU8sBqnC21 `Amg˷gꄤ20y2Yf!"-PbD-ix:, b&Wrqlmɓ$F)1rCK4 g#niibFFSL<$)ZƤ̌R$c?NfYRt$VqQ1@AJђ\!4ɬ,*ES[%G1IkH jd PTzp.s-.u$=d~M/$]s{@N% t)\CTh?Ή -FJ"WeC+Ut m濋N`7xMxUOH%V.z`ƧYG|nfn7.47S蹉(rW!_6 bRLsVYŊl^&7"Ѣocᗗv{+Vf Sz^^.|^޾4qx[_^>m6Ѹ^?tr yxr}¿nȟ;geKA]}!~ ]M endstream endobj 24 0 obj(http://www.managementhelp.org/searchbox-logo-126x32.gif) endobj 25 0 obj(ng+{jj) endobj 26 0 obj<> endobj 27 0 obj<> endobj 28 0 obj<> endobj 29 0 obj<>>> endobj 30 0 obj<> endobj 31 0 obj<><><><><><><><><><><><><><>]/C/SpdrArt/P 30 0 R/S/Article/Pg 32 0 R>> endobj 32 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 0>> endobj 33 0 obj<> endobj 34 0 obj[32 0 R/XYZ 0 747.0 null] endobj 35 0 obj<> endobj 36 0 obj[31 0 R] endobj 37 0 obj<>/Subtype/Link/A<>>> endobj 38 0 obj<>/Font<>>>/DA(/Helv 0 Tf 0 g )>> endobj 39 0 obj<> endobj 40 0 obj<> endobj 41 0 obj<> endobj 42 0 obj[37 0 R 43 0 R 44 0 R 45 0 R 46 0 R 47 0 R 48 0 R 49 0 R 50 0 R 51 0 R 52 0 R 53 0 R 54 0 R 55 0 R 56 0 R 57 0 R 58 0 R 59 0 R 60 0 R 61 0 R 62 0 R 63 0 R 64 0 R 65 0 R 66 0 R 67 0 R 68 0 R 69 0 R 70 0 R 71 0 R] endobj 43 0 obj<>/Subtype/Link/A<>>> endobj 44 0 obj<>/Subtype/Link/A<>>> endobj 45 0 obj<>/Subtype/Link/A<>>> endobj 46 0 obj<>/Subtype/Link/A<>>> endobj 47 0 obj<>/Subtype/Link/A<>>> endobj 48 0 obj<>/Subtype/Link/A<>>> endobj 49 0 obj<>/Subtype/Link/A<>>> endobj 50 0 obj<>/Subtype/Link/A<>>> endobj 51 0 obj<>/Subtype/Link/A<>>> endobj 52 0 obj<>/Subtype/Link/A<>>> endobj 53 0 obj<>/Subtype/Link/A<>>> endobj 54 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 55 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 56 0 obj<>/Subtype/Link/A<>>> endobj 57 0 obj<>/Subtype/Link/A<>>> endobj 58 0 obj<>/Subtype/Link/A<>>> endobj 59 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 60 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 61 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 62 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 63 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 64 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 65 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 66 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 67 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 68 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 69 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 70 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 71 0 obj<>/Subtype/Link/A<>>> endobj 72 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 1>> endobj 73 0 obj[31 0 R] endobj 74 0 obj<>/Subtype/Link/A<>>> endobj 75 0 obj[74 0 R 76 0 R 77 0 R] endobj 76 0 obj<>/Subtype/Link/A<>>> endobj 77 0 obj<>/Subtype/Link/A<>>> endobj 78 0 obj[72 0 R/XYZ 0 301.200012 null] endobj 79 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 2>> endobj 80 0 obj[31 0 R] endobj 81 0 obj[79 0 R/XYZ 0 225.399994 null] endobj 82 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 3>> endobj 83 0 obj[31 0 R] endobj 84 0 obj[82 0 R/XYZ 0 497.799988 null] endobj 85 0 obj[82 0 R/XYZ 0 69.9077 null] endobj 86 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 4>> endobj 87 0 obj[31 0 R] endobj 88 0 obj[86 0 R/XYZ 0 366.323059 null] endobj 89 0 obj[86 0 R/XYZ 0 44.92308 null] endobj 90 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 5>> endobj 91 0 obj[31 0 R] endobj 92 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 93 0 obj[92 0 R] endobj 94 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 6>> endobj 95 0 obj[31 0 R] endobj 96 0 obj<>/Subtype/Link/A<>>> endobj 97 0 obj[96 0 R] endobj 98 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 7>> endobj 99 0 obj[31 0 R] endobj 100 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 101 0 obj[100 0 R 102 0 R 103 0 R 104 0 R] endobj 102 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 103 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 104 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 105 0 obj[98 0 R/XYZ 0 369.399994 null] endobj 106 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 8>> endobj 107 0 obj[31 0 R] endobj 108 0 obj<>/Subtype/Link/A<>>> endobj 109 0 obj[108 0 R 110 0 R] endobj 110 0 obj<>/Subtype/Link/A<>>> endobj 111 0 obj[106 0 R/XYZ 0 102.046158 null] endobj 112 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 9>> endobj 113 0 obj[31 0 R] endobj 114 0 obj<>/Subtype/Link/A<>>> endobj 115 0 obj[114 0 R 116 0 R 117 0 R 118 0 R 119 0 R 120 0 R] endobj 116 0 obj<>/Subtype/Link/A<>>> endobj 117 0 obj<>/Subtype/Link/A<>>> endobj 118 0 obj<>/Subtype/Link/A<>>> endobj 119 0 obj<>/Subtype/Link/A<>>> endobj 120 0 obj<>/Subtype/Link/A<>>> endobj 121 0 obj[112 0 R/XYZ 0 563.646118 null] endobj 122 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 10>> endobj 123 0 obj<> endobj 124 0 obj<> endobj 125 0 obj<> endobj 126 0 obj<> endobj 127 0 obj<> endobj 130 0 obj[31 0 R] endobj 131 0 obj[122 0 R/XYZ 0 727.861572 null] endobj 132 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 11>> endobj 133 0 obj<> endobj 136 0 obj[31 0 R] endobj 137 0 obj[132 0 R/XYZ 0 629.400024 null] endobj 138 0 obj[132 0 R/XYZ 0 370.461548 null] endobj 139 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 140 0 obj[139 0 R 141 0 R] endobj 141 0 obj<>/Subtype/Link/A<>/PA<>>> endobj 142 0 obj[132 0 R/XYZ 0 297.061523 null] endobj 143 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 12>> endobj 146 0 obj[31 0 R] endobj 147 0 obj[143 0 R/XYZ 0 663.0 null] endobj 148 0 obj<>/ProcSet[/PDF/Text]>>/StructParents 13>> endobj 151 0 obj[31 0 R] endobj 152 0 obj[148 0 R/XYZ 0 696.599976 null] endobj 153 0 obj[148 0 R/XYZ 0 490.215393 null] endobj 154 0 obj<>/Subtype/Link/A<>>> endobj 155 0 obj[154 0 R 157 0 R 158 0 R 159 0 R 160 0 R 161 0 R 162 0 R 163 0 R 164 0 R 165 0 R 166 0 R 167 0 R] endobj 156 0 obj[148 0 R/XYZ 0 433.030762 null] endobj 157 0 obj<>/Subtype/Link/A<>>> endobj 158 0 obj<>/Subtype/Link/A<>>> endobj 159 0 obj<>/Subtype/Link/A<>>> endobj 160 0 obj<>/Subtype/Link/A<>>> endobj 161 0 obj<>/Subtype/Link/A<>>> endobj 162 0 obj<>/Subtype/Link/A<>>> endobj 163 0 obj<>/Subtype/Link/A<>>> endobj 164 0 obj<>/Subtype/Link/A<>>> endobj 165 0 obj<>/Subtype/Link/A<>>> endobj 166 0 obj<>/Subtype/Link/A<>>> endobj 167 0 obj<>/Subtype/Link/A<>>> endobj 168 0 obj<>/Font<>/XObject<>/ProcSet[/PDF/Text/ImageC/ImageI]>>/StructParents 14>> endobj 171 0 obj[31 0 R] endobj 172 0 obj<>/FT/Tx/Subtype/Widget/P 168 0 R/T(f1)/AA<>/AP<>/DA(/Helv 12 Tf 0 g)/MK<>/TM(keyword)>> endobj 173 0 obj[172 0 R 179 0 R 180 0 R 181 0 R 186 0 R 187 0 R 188 0 R 189 0 R] endobj 174 0 obj<> endobj 175 0 obj<> endobj 176 0 obj<> endobj 177 0 obj<> endobj 178 0 obj<>/Subtype/Form/FormType 1/Matrix[1.0 0.0 0.0 1.0 0.0 0.0]>>stream 1 g 0 0 72 22 re f 0.501953 g 1 1 m 1 21 l 71 21 l 70 20 l 2 20 l 2 2 l f 0.75293 g 71 21 m 71 1 l 1 1 l 2 2 l 70 2 l 70 20 l f 0 G 0.5 0.5 71 21 re s endstream endobj 179 0 obj<>/TM(mode)>> endobj 180 0 obj<>/TM(tag)>> endobj 181 0 obj<>/Ff 65540/MK<>/TP 1>>/TM(Submit)>> endobj 182 0 obj 29 endobj 183 0 obj<>/XObject<>/ProcSet[/PDF/ImageC/ImageI]>>/Subtype/Form>>stream q 21 0 0 21 0 0 cm /Im0 Do Q endstream endobj 184 0 obj<>/ProcSet[/PDF]>>/Subtype/Form/FormType 1/Matrix[1.0 0.0 0.0 1.0 0.0 0.0]>>stream q 1 1 23 23 re W n q 2 2 21 21 re W n 1 0 0 1 2 2 cm /FRM Do Q Q endstream endobj 185 0 obj<>/Flags 30/S/SubmitForm/Fields[(KAFLMDKJHGPILLIPFJONCIAHBANMOKCE.form1.x)]>> endobj 186 0 obj<>/Subtype/Link/A<>>> endobj 187 0 obj<>/Subtype/Link/A<>>> endobj 188 0 obj<>/Subtype/Link/A<>>> endobj 189 0 obj<>/Subtype/Link/A<>>> endobj 208 0 obj 6694 endobj 209 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_1 1 Tf 19.38461 0 0 19.38461 158.45398 730.35077 Tm (Basic Guide to Program Evaluation)Tj /T1_0 1 Tf 12.92308 0 0 12.92308 64.76492 712.63898 Tm (Written by )Tj ET 0 0 1 RG 0.65132 w 10 M 0 j 0 J []0 d 123.62953 710.68506 m 282.62213 710.68506 l S 0 0 1 rg BT /T1_0 1 Tf 12.92308 0 0 12.92308 123.62953 712.63898 Tm (Carter McNamara, MBA, PhD)Tj 0 0 0 rg ( | Applies to nonprofits and for-profits unless noted)Tj ET 84.92493 693.17731 m 149.54031 693.17731 l S 0 0 1 rg BT /T1_0 1 Tf 12.92308 0 0 12.92308 84.92493 695.13129 Tm (First-Timers)Tj 0 0 0 rg ( | )Tj ET 158.58646 693.17731 m 240.77724 693.17731 l S 0 0 1 rg BT /T1_0 1 Tf 12.92308 0 0 12.92308 158.58646 695.13129 Tm (Library Catalog)Tj 0 0 0 rg ( | )Tj ET 249.82338 693.17731 m 351.74768 693.17731 l S 0 0 1 rg BT /T1_0 1 Tf 12.92308 0 0 12.92308 249.82338 695.13129 Tm (Library Home Page)Tj 0 0 0 rg ( | )Tj ET 360.79385 693.17731 m 418.58585 693.17731 l S 0 0 1 rg BT /T1_0 1 Tf 12.92308 0 0 12.92308 360.79385 695.13129 Tm (Contact Us)Tj 0 0 0 rg ( | )Tj ET 427.63202 693.17731 m 527.07507 693.17731 l S 0 0 1 rg BT /T1_0 1 Tf 12.92308 0 0 12.92308 427.63202 695.13129 Tm (Reprint Permission)Tj ET 162.01753 675.66962 m 349.76401 675.66962 l S BT /T1_0 1 Tf 12.92308 0 0 12.92308 162.01753 677.6236 Tm (Authenticity Consulting Home Page)Tj 0 0 0 rg ( | )Tj ET 358.81015 675.66962 m 449.98245 675.66962 l S 0 0 1 rg BT /T1_0 1 Tf 12.92308 0 0 12.92308 358.81015 677.6236 Tm (MAP Home Page)Tj 0 0 0 rg 14 0 0 14 10 640.19092 Tm (\(Much of the following information is adapted from the guidebook,)Tj /T1_1 1 Tf ( )Tj ET 0.7056 w 391.45798 638.0741 m 586.28204 638.0741 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 391.45798 640.19092 Tm (Field Guide to Nonprofit Program )Tj ET 10 619.27411 m 206.36401 619.27411 l S BT /T1_0 1 Tf 14 0 0 14 10 621.39093 Tm (Design, Marketing and Evaluation.)Tj 0 0 0 rg (\))Tj 0 -2.7 TD (This document provides guidance toward planning and implementing an eval\ uation process for for-)Tj 0 -1.2 TD (profit or nonprofit programs -- there are many kinds of evaluations that\ can be applied to programs, for )Tj T* (example, goals-based, process-based, outcomes-based, etc.)Tj 0 -2.55714 TD (NOTE: Nonprofit organizations are increasingly interested in outcomes-ba\ sed evaluation. If you are )Tj 0 -1.2 TD (interested in learning more about outcomes-based evaluation, then see )Tj ET 406.59201 495.27414 m 589.73999 495.27414 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 406.59201 497.39093 Tm (Outcomes-Based Evaluations in )Tj ET 10 476.47412 m 147.25601 476.47412 l S BT /T1_0 1 Tf 14 0 0 14 10 478.59094 Tm (Nonprofit Organizations)Tj 0 0 0 rg (.)Tj 0 -2.7 TD (NOTE: Be sure to see )Tj ET 136.36401 438.67413 m 257.29599 438.67413 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 136.36401 440.79092 Tm (Additional Resources)Tj 0 0 0 rg ( and )Tj ET 284.51199 438.67413 m 408.16 438.67413 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 284.51199 440.79092 Tm (Related Library Links)Tj 0 0 0 rg ( at the end of this document. )Tj -19.608 -1.34286 Td (Also, you might benefit from review of the )Tj ET 254.58 419.87415 m 456.38998 419.87415 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 254.58 421.99094 Tm (Program Planning and Management)Tj 0 0 0 rg (. Lastly, use the )Tj ET 548.15997 419.87415 m 599.87598 419.87415 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 548.15997 421.99094 Tm (Program )Tj ET 10 401.07413 m 180.33801 401.07413 l S BT /T1_0 1 Tf 14 0 0 14 10 403.19092 Tm (Evaluation Planning Checklist)Tj 0 0 0 rg ( to help you plan evaluations for your programs.)Tj /T1_1 1 Tf 16.15384 0 0 16.15384 10 363.54102 Tm (Table of Contents)Tj ET 1.1676 w 10 342.50351 m 521.36401 342.50351 l S 0 0 1 rg BT /T1_1 1 Tf 14 0 0 14 10 346.00632 Tm (Program Evaluation: carefully getting information to make decisions abou\ t programs)Tj ET 10 323.70352 m 237.89201 323.70352 l S BT /T1_1 1 Tf 14 0 0 14 10 327.20633 Tm (Where Program Evaluation is Helpful)Tj ET 10 304.9035 m 117.33801 304.9035 l S 0.7056 w 117.33801 306.28952 m 357.21402 306.28952 l S BT /T1_1 1 Tf 14 0 0 14 10 308.40631 Tm (Basic Ingredients )Tj /T1_0 1 Tf (\(you need an organization and program\(s\)\))Tj ET 1.1676 w 10 286.10352 m 194.35201 286.10352 l S 0.7056 w 194.35201 287.4895 m 500.35001 287.4895 l S BT /T1_1 1 Tf 14 0 0 14 10 289.60632 Tm (Planning Program Evaluation:)Tj /T1_0 1 Tf (\(what do you want to learn about, what info is needed\))Tj ET 1.1676 w 10 267.30353 m 228.53998 267.30353 l S 0.7056 w 228.53998 268.68951 m 531.80804 268.68951 l S BT /T1_1 1 Tf 14 0 0 14 10 270.8063 Tm (Major Types of Program Evaluation)Tj /T1_0 1 Tf ( \(evaluating program processes, goals, outcomes, etc.\))Tj ET 1.1676 w 10 248.50351 m 279.06598 248.50351 l S 0.7056 w 279.06598 249.88953 m 540.34802 249.88953 l S BT /T1_1 1 Tf 14 0 0 14 10 252.00632 Tm (Overview of Methods to Collect Information )Tj /T1_0 1 Tf (\(questionnaires, interviews, focus groups, etc.\))Tj ET 1.1676 w 10 229.70352 m 206.37801 229.70352 l S 0.7056 w 206.37801 231.08951 m 552.80804 231.08951 l S BT /T1_1 1 Tf 14 0 0 14 10 233.20633 Tm (Selecting Which Methods to Use )Tj /T1_0 1 Tf (\(which methods work best to get needed info from audiences\))Tj ET 1.1676 w 10 210.90352 m 250.73 210.90352 l S 0.7056 w 250.73 212.28952 m 504.63403 212.28952 l S BT /T1_1 1 Tf 14 0 0 14 10 214.40633 Tm (Analyzing and Interpreting Information)Tj /T1_0 1 Tf ( \(to make conclusions and recommendations\))Tj ET 1.1676 w 10 192.10353 m 186.56798 192.10353 l S BT /T1_1 1 Tf 14 0 0 14 10 195.60631 Tm (Reporting Evaluation Results)Tj ET 10 173.30353 m 249.20398 173.30353 l S BT /T1_1 1 Tf 14 0 0 14 10 176.80632 Tm (Who Should Carry Out the Evaluation?)Tj ET 10 154.50351 m 196.69 154.50351 l S BT /T1_1 1 Tf 14 0 0 14 10 158.00633 Tm (Contents of an Evaluation Plan)Tj ET 10 135.70352 m 106.44601 135.70352 l S BT /T1_1 1 Tf 14 0 0 14 10 139.20633 Tm (Pitfalls to Avoid)Tj 0 0 0 rg /T1_2 1 Tf T* (To Form Local Learning Communities to Design and Implement Evaluation Pl\ ans)Tj ET 0.7056 w 10 82.48952 m 366.57999 82.48952 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 10 84.60632 Tm (To Form Local Learning Communities, eg, to Learn This Topic)Tj 0 0 0 rg 0 -1.34286 TD (\(at http://www.authenticityconsulting.com/pubs.htm\))Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(1 of 15\)10/12/200\ 5 5:05:24 AM)Tj ET EMC endstream endobj 210 0 obj 4957 endobj 211 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_1 1 Tf 14 0 0 14 10 753.97552 Tm (Free, Complete, On-line Training Programs That Include This Topic!)Tj /T1_0 1 Tf 0 -1.2 TD (This topic is also included in the )Tj ET 0 0 1 RG 0.7056 w 10 M 0 j 0 J []0 d 195.5 735.05872 m 359.57999 735.05872 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 195.5 737.17554 Tm (Free Nonprofit Micro-eMBA)Tj 0 0 0 rg ( learning module, )Tj ET 461.45798 735.05872 m 554.78204 735.05872 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 461.45798 737.17554 Tm (Designing Your )Tj ET 10 716.25873 m 382.90399 716.25873 l S BT /T1_0 1 Tf 14 0 0 14 10 718.37555 Tm (Program Evaluation Plans \(including outcomes-based evaluations\))Tj 0 0 0 rg (. This complete, "nuts and bolts", free )Tj 0 -1.34286 TD (training program is geared to leaders, managers, consultants and volunte\ ers who serve nonprofit )Tj 0 -1.2 TD (organizations.)Tj /T1_2 1 Tf 0 -2.55714 TD (Tell Friends! Local Professional Organizations! Spread the Word!)Tj /T1_0 1 Tf 0 -1.2 TD (Tell friends and professional organizations about these free programs! A\ dvertise them in your )Tj T* (newsletters and web sites so that others can save training dollars, too!\ )Tj ET 0.5 0.5 0.5 rg 10 580.59998 m 10 582.59998 l 602 582.59998 l 601 581.59998 l 11 581.59998 l 11 581.59998 l h f 0.875 0.875 0.875 rg 602 582.59998 m 602 580.59998 l 10 580.59998 l 11 581.59998 l 601 581.59998 l 601 581.59998 l h f 0 0 0 rg BT /T1_3 1 Tf 14 0 0 14 10 561.57556 Tm (A Brief Introduction ...)Tj /T1_0 1 Tf T* (Note that the concept of program evaluation can include a wide variety o\ f methods to evaluate many )Tj T* (aspects of programs in nonprofit or for-profit organizations. There are \ numerous books and other )Tj T* (materials that provide in-depth analysis of evaluations, their designs, \ methods, combination of methods )Tj T* (and techniques of analysis. However, personnel do not have to be experts\ in these topics to carry out a )Tj T* (useful program evaluation. The "20-80" rule applies here, that 20% of ef\ fort generates 80% of the )Tj T* (needed results. It's better to do what might turn out to be an average e\ ffort at evaluation than to do no )Tj T* (evaluation at all. \(Besides, if you resort to bringing in an evaluation\ consultant, you should be a smart )Tj T* (consumer. Far too many program evaluations generate information that is \ either impractical or irrelevant )Tj T* (-- if the information is understood at all.\) This document orients pers\ onnel to the nature of program )Tj T* (evaluation and how it can be carried out in a realistic and practical fa\ shion. )Tj 0 -2.55714 TD (Note that much of the information in this section was gleaned from vario\ us works of Michael Quinn )Tj 0 -1.2 TD (Patton.)Tj ET 0.5 0.5 0.5 rg 10 308.20001 m 10 310.20001 l 602 310.20001 l 601 309.20001 l 11 309.20001 l 11 309.20001 l h f 0.875 0.875 0.875 rg 602 310.20001 m 602 308.20001 l 10 308.20001 l 11 309.20001 l 601 309.20001 l 601 309.20001 l h f 0 0 0 rg BT /T1_3 1 Tf 16.15384 0 0 16.15384 10 287.32562 Tm (Program Evaluation)Tj /T1_0 1 Tf 14 0 0 14 151.34616 287.32562 Tm ( \(by Carter McNamara, PhD; last revision: Feb 16, 1998\))Tj /T1_1 1 Tf -10.09615 -1.25249 Td (Some Myths About Program Evaluation)Tj /T1_0 1 Tf T* (1.. Many people believe evaluation is a useless activity that generates \ lots of boring data with useless )Tj T* (conclusions. This was a problem with evaluations in the past when progra\ m evaluation methods were )Tj T* (chosen largely on the basis of achieving complete scientific accuracy, r\ eliability and validity. This )Tj T* (approach often generated extensive data from which very carefully chosen\ conclusions were drawn. )Tj T* (Generalizations and recommendations were avoided. As a result, evaluatio\ n reports tended to reiterate )Tj T* (the obvious and left program administrators disappointed and skeptical a\ bout the value of evaluation in )Tj T* (general. More recently \(especially as a result of Michael Patton's deve\ lopment of utilization-focused )Tj T* (evaluation\), evaluation has focused on utility, relevance and practical\ ity at least as much as scientific )Tj T* (validity. )Tj 0 -2.55714 TD (2. Many people believe that evaluation is about proving the success or f\ ailure of a program. This myth )Tj 0 -1.2 TD (assumes that success is implementing the perfect program and never havin\ g to hear from employees, )Tj T* (customers or clients again -- the program will now run itself perfectly.\ This doesn't happen in real life. )Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(2 of 15\)10/12/200\ 5 5:05:24 AM)Tj ET EMC endstream endobj 212 0 obj 4664 endobj 213 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_0 1 Tf 14 0 0 14 10 753.97552 Tm (Success is remaining open to continuing feedback and adjusting the progr\ am accordingly. Evaluation )Tj 0 -1.2 TD (gives you this continuing feedback.)Tj 0 -2.55714 TD (3. Many believe that evaluation is a highly unique and complex process t\ hat occurs at a certain time in a )Tj 0 -1.2 TD (certain way, and almost always includes the use of outside experts. Many\ people believe they must )Tj T* (completely understand terms such as validity and reliability. They don't\ have to. They do have to )Tj T* (consider what information they need in order to make current decisions a\ bout program issues or needs. )Tj T* (And they have to be willing to commit to understanding what is really go\ ing on. Note that many people )Tj T* (regularly undertake some nature of program evaluation -- they just don't\ do it in a formal fashion so they )Tj T* (don't get the most out of their efforts or they make conclusions that ar\ e inaccurate \(some evaluators )Tj T* (would disagree that this is program evaluation if not done methodically\)\ . Consequently, they miss )Tj T* (precious opportunities to make more of difference for their customer and\ clients, or to get a bigger bang )Tj T* (for their buck.)Tj /T1_1 1 Tf 0 -2.55714 TD (So What is Program Evaluation? )Tj /T1_0 1 Tf 0 -1.2 TD (First, we'll consider "what is a program?" Typically, organizations work\ from their mission to identify )Tj T* (several overall goals which must be reached to accomplish their mission.\ In nonprofits, each of these )Tj T* (goals often becomes a program. Nonprofit programs are organized methods \ to provide certain related )Tj T* (services to constituents, e.g., clients, customers, patients, etc. Progr\ ams must be evaluated to decide if )Tj T* (the programs are indeed useful to constituents. In a for-profit, a progr\ am is often a one-time effort to )Tj T* (produce a new product or line of products. )Tj 0 -2.2 TD (So, still, what is program evaluation? Program evaluation is carefully c\ ollecting information about a )Tj 0 -1.2 TD (program or some aspect of a program in order to make necessary decisions\ about the program. Program )Tj T* (evaluation can include any or a variety of at least 35 different types o\ f evaluation, such as for needs )Tj T* (assessments, accreditation, cost/benefit analysis, effectiveness, effici\ ency, formative, summative, goal-)Tj T* (based, process, outcomes, etc. The type of evaluation you undertake to i\ mprove your programs depends )Tj T* (on what you want to learn about the program. Don't worry about what type\ of evaluation you need or are )Tj T* (doing -- worry about what you need to know to make the program decisions\ you need to make, and )Tj T* (worry about how you can accurately collect and understand that informati\ on.)Tj ET 0.5 0.5 0.5 rg 10 232.39998 m 10 234.39998 l 602 234.39998 l 601 233.39998 l 11 233.39998 l 11 233.39998 l h f 0.875 0.875 0.875 rg 602 234.39998 m 602 232.39998 l 10 232.39998 l 11 233.39998 l 601 233.39998 l 601 233.39998 l h f 0 0 0 rg BT /T1_2 1 Tf 16.15384 0 0 16.15384 10 211.52562 Tm (Where Program Evaluation is Helpful )Tj /T1_0 1 Tf 14 0 0 14 276.99075 211.52562 Tm (\(by Carter McNamara, PhD; last revision: Feb 16, 1998\))Tj /T1_1 1 Tf -19.07077 -1.25249 Td (Frequent Reasons:)Tj /T1_0 1 Tf T* (Program evaluation can:)Tj T* (1. Understand, verify or increase the impact of products or services on \ customers or clients - These )Tj T* ("outcomes" evaluations are increasingly required by nonprofit funders as\ verification that the nonprofits )Tj T* (are indeed helping their constituents. Too often, service providers \(fo\ r-profit or nonprofit\) rely on their )Tj T* (own instincts and passions to conclude what their customers or clients r\ eally need and whether the )Tj T* (products or services are providing what is needed. Over time, these orga\ nizations find themselves in a )Tj T* (lot of guessing about what would be a good product or service, and trial\ and error about how new )Tj T* (products or services could be delivered.)Tj T* (2. Improve delivery mechanisms to be more efficient and less costly - Ov\ er time, product or service )Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(3 of 15\)10/12/200\ 5 5:05:24 AM)Tj ET EMC endstream endobj 214 0 obj 4239 endobj 215 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_0 1 Tf 14 0 0 14 10 753.97552 Tm (delivery ends up to be an inefficient collection of activities that are \ less efficient and more costly than )Tj 0 -1.2 TD (need be. Evaluations can identify program strengths and weaknesses to im\ prove the program.)Tj T* (3. Verify that you're doing what you think you're doing - Typically, pla\ ns about how to deliver services, )Tj T* (end up changing substantially as those plans are put into place. Evaluat\ ions can verify if the program is )Tj T* (really running as originally planned.)Tj /T1_1 1 Tf 0 -2.2 TD (Other Reasons:)Tj /T1_0 1 Tf 0 -1.2 TD (Program evaluation can:)Tj T* (4. Facilitate management's really thinking about what their program is a\ ll about, including its goals, how )Tj T* (it meets it goals and how it will know if it has met its goals or not.)Tj T* (5. Produce data or verify results that can be used for public relations \ and promoting services in the )Tj T* (community. )Tj T* (6. Produce valid comparisons between programs to decide which should be \ retained, e.g., in the face of )Tj T* (pending budget cuts.)Tj T* (7. Fully examine and describe effective programs for duplication elsewhe\ re. )Tj /T1_2 1 Tf 16.15384 0 0 16.15384 10 483.92563 Tm (Basic Ingredients: Organization and Program\(s\) )Tj /T1_0 1 Tf 12.92308 0 0 12.92308 347.87384 483.92563 Tm (\(by Carter McNamara, PhD; last revision: Feb )Tj -26.14499 -1.28528 Td (16, 1998\))Tj /T1_1 1 Tf 14 0 0 14 10 450.88324 Tm (You Need An Organization:)Tj /T1_0 1 Tf T* (This may seem too obvious to discuss, but before an organization embarks\ on evaluating a program, it )Tj T* (should have well established means to conduct itself as an organization,\ e.g., \(in the case of a nonprofit\) )Tj T* (the board should be in good working order, the organization should be st\ affed and organized to conduct )Tj T* (activities to work toward the mission of the organization, and there sho\ uld be no current crisis that is )Tj T* (clearly more important to address than evaluating programs.)Tj /T1_1 1 Tf 0 -2.55714 TD (You Need Program\(s\):)Tj /T1_0 1 Tf 0 -1.2 TD (To effectively conduct program evaluation, you should first have program\ s. That is, you need a strong )Tj T* (impression of what your customers or clients actually need. \(You may ha\ ve used a needs assessment to )Tj T* (determine these needs -- itself a form of evaluation, but usually the fi\ rst step in a good marketing plan\). )Tj T* (Next, you need some effective methods to meet each of those goals. These\ methods are usually in the )Tj T* (form of programs.)Tj 0 -2.55714 TD (It often helps to think of your programs in terms of inputs, process, ou\ tputs and outcomes. Inputs are the )Tj 0 -1.2 TD (various resources needed to run the program, e.g., money, facilities, cu\ stomers, clients, program staff, )Tj T* (etc. The process is how the program is carried out, e.g., customers are \ served, clients are counseled, )Tj T* (children are cared for, art is created, association members are supporte\ d, etc. The outputs are the units of )Tj T* (service, e.g., number of customers serviced, number of clients counseled\ , children cared for, artistic )Tj T* (pieces produced, or members in the association. Outcomes are the impacts\ on the customers or on clients )Tj T* (receiving services, e.g., increased mental health, safe and secure devel\ opment, richer artistic )Tj T* (appreciation and perspectives in life, increased effectiveness among mem\ bers, etc.)Tj /T1_2 1 Tf 16.15384 0 0 16.15384 10 56.03334 Tm (Planning Your Program Evaluation )Tj /T1_0 1 Tf 14 0 0 14 261.30539 56.03334 Tm (\(by Carter McNamara, PhD\))Tj /T1_1 1 Tf -17.95038 -1.25249 Td (Depends on What Information You Need to Make Your Decisions and On Your \ Resources:)Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(4 of 15\)10/12/200\ 5 5:05:24 AM)Tj ET EMC endstream endobj 216 0 obj 4077 endobj 217 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_0 1 Tf 14 0 0 14 10 751.69861 Tm (Often, management wants to know everything about their products, service\ s or programs. However, )Tj 0 -1.2 TD (limited resources usually force managers to prioritize what they need to\ know to make current decisions.)Tj 0 -2.55714 TD (Your program evaluation plans depend on what information you need to col\ lect in order to make major )Tj 0 -1.2 TD (decisions. Usually, management is faced with having to make major decisi\ ons due to decreased funding, )Tj T* (ongoing complaints, unmet needs among customers and clients, the need to\ polish service delivery, etc. )Tj T* (For example, do you want to know more about what is actually going on in\ your programs, whether your )Tj T* (programs are meeting their goals, the impact of your programs on custome\ rs, etc? You may want other )Tj T* (information or a combination of these. Ultimately, it's up to you.)Tj 0 -2.55714 TD (But the more focused you are about what you want to examine by the evalu\ ation, the more efficient you )Tj 0 -1.2 TD (can be in your evaluation, the shorter the time it will take you and ult\ imately the less it will cost you )Tj T* (\(whether in your own time, the time of your employees and/or the time o\ f a consultant\).)Tj 0 -2.55714 TD (There are trade offs, too, in the breadth and depth of information you g\ et. The more breadth you want, )Tj 0 -1.2 TD (usually the less depth you get \(unless you have a great deal of resourc\ es to carry out the evaluation\). On )Tj T* (the other hand, if you want to examine a certain aspect of a program in \ great detail, you will likely not )Tj T* (get as much information about other aspects of the program.)Tj 0 -2.55714 TD (For those starting out in program evaluation or who have very limited re\ sources, they can use various )Tj 0 -1.2 TD (methods to get a good mix of breadth and depth of information. They can \ both understand more about )Tj T* (certain areas of their programs and not go bankrupt doing so.)Tj /T1_1 1 Tf 0 -2.55714 TD (Key Considerations:)Tj /T1_0 1 Tf 0 -1.20001 TD (Consider the following key questions when designing a program evaluation\ .)Tj 0 -1.2 TD (1. For what purposes is the evaluation being done, i.e., what do you wan\ t to be able to decide as a result )Tj T* (of the evaluation?)Tj T* (2. Who are the audiences for the information from the evaluation, e.g., \ customers, bankers, funders, )Tj T* (board, management, staff, customers, clients, etc.)Tj 0 -1.20001 TD (3. What kinds of information are needed to make the decision you need to\ make and/or enlighten your )Tj 0 -1.2 TD (intended audiences, e.g., information to really understand the process o\ f the product or program \(its )Tj T* (inputs, activities and outputs\), the customers or clients who experienc\ e the product or program, strengths )Tj T* (and weaknesses of the product or program, benefits to customers or clien\ ts \(outcomes\), how the product )Tj T* (or program failed and why, etc.)Tj 0 -1.20001 TD (4. From what sources should the information be collected, e.g., employee\ s, customers, clients, groups of )Tj 0 -1.2 TD (customers or clients and employees together, program documentation, etc.\ )Tj T* (5. How can that information be collected in a reasonable fashion, e.g., \ questionnaires, interviews, )Tj T* (examining documentation, observing customers or employees, conducting fo\ cus groups among )Tj T* (customers or employees, etc.)Tj 0 -1.20001 TD (6. When is the information needed \(so, by when must it be collected\)?)Tj 0 -1.2 TD (7. What resources are available to collect the information?)Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(5 of 15\)10/12/200\ 5 5:05:24 AM)Tj ET EMC endstream endobj 218 0 obj 2428 endobj 219 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /WebCaptureBG BMC /WebCaptureFN <>BDC q 0 18 612 756 re W* n /Artifact <>BDC Q 1 1 1 rg 10 665.07692 128 92 re f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0 0 0 rg 10 757.07692 m 138 757.07692 l 137 756.07692 l 11 756.07692 l h f 10 757.07692 m 10 665.07692 l 11 666.07692 l 11 756.07692 l h f 138 757.07692 m 138 665.07692 l 137 666.07692 l 137 756.07692 l h f 10 665.07692 m 138 665.07692 l 137 666.07692 l 11 666.07692 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 1 1 1 rg 11 736.07692 97.85106 20 re f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 108.85106 736.07692 28.14894 20 re f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 11 706.07692 97.85106 30 re f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 108.85106 706.07692 28.14894 30 re f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0 0 0 rg 11 666.07692 126 40 re f EMC EMC EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_0 1 Tf 14 0 0 14 23.57454 740.85248 Tm (Search Now: )Tj 5.20364 -1.69826 Td ( )Tj 2.38754 0.34825 Td ( )Tj ET q 126 0 0 36 11 668.07692 cm /Im0 Do Q BT /T1_0 1 Tf 14 0 0 14 84.534 653.05247 Tm (Carter McNamara, MBA, PhD, Copyright 1999, )Tj ET 0 0 1 RG 0.7056 w 10 M 0 j 0 J []0 d 359.06 650.93568 m 527.466 650.93568 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 359.06 653.05247 Tm (Authenticity Consulting, LLC)Tj ET 135.89301 632.13568 m 476.10699 632.13568 l S BT /T1_0 1 Tf 14 0 0 14 135.89301 634.25247 Tm (Library and its contents are not to be used to generate profits)Tj ET 0.65132 w 256.6403 614.42346 m 355.3597 614.42346 l S BT /T1_0 1 Tf 12.92308 0 0 12.92308 256.6403 616.37744 Tm (Reprint permission)Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(15 of 15\)10/12/20\ 05 5:05:24 AM)Tj ET EMC endstream endobj 220 0 obj 4496 endobj 221 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_1 1 Tf 16.15384 0 0 16.15384 10 752.12561 Tm (Some Major Types of Program Evaluation)Tj 14 0 0 14 303.8869 752.12561 Tm ( )Tj /T1_0 1 Tf 12.92308 0 0 12.92308 307.3869 752.12561 Tm (\(by Carter McNamara, PhD; last revision: Feb 16, 1998\))Tj 14 0 0 14 10 734.59094 Tm (When designing your evaluation approach, it may be helpful to review the\ following three types of )Tj 0 -1.2 TD (evaluations, which are rather common in organizations. Note that you sho\ uld not design your evaluation )Tj T* (approach simply by choosing which of the following three types you will \ use -- you should design your )Tj T* (evaluation approach by carefully addressing the above )Tj ET 0 0 1 RG 0.7056 w 10 M 0 j 0 J []0 d 318.30801 682.0741 m 422.90198 682.0741 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 318.30801 684.19092 Tm (key considerations)Tj 0 0 0 rg (.)Tj /T1_2 1 Tf -22.022 -2.7 Td (Goals-Based Evaluation \(are your programs achieving their overall, pred\ etermined objectives?\))Tj /T1_0 1 Tf T* (Often programs are established to meet one or more specific goals. These\ goals are often described in )Tj T* (the original program plans.)Tj 0 -2.55714 TD (Goal-based evaluations are evaluating the extent to which programs are m\ eeting predetermined goals or )Tj 0 -1.2 TD (objectives. Questions to ask yourself when designing an evaluation to se\ e if you reached your goals, are:)Tj T* (1. How were the program goals \(and objectives, is applicable\) establis\ hed? Was the process effective?)Tj T* (2. What is the status of the program's progress toward achieving the goa\ ls? )Tj T* (3. Will the goals be achieved according to the timelines specified in th\ e program implementation or )Tj T* (operations plan? If not, then why?)Tj T* (4. Do personnel have adequate resources \(money, equipment, facilities, \ training, etc.\) to achieve the )Tj T* (goals?)Tj T* (5. How should priorities be changed to put more focus on achieving the g\ oals? \(Depending on the )Tj T* (context, this question might be viewed as a program management decision,\ more than an evaluation )Tj T* (question.\))Tj T* (6. How should timelines be changed \(be careful about making these chang\ es - know why efforts are )Tj T* (behind schedule before timelines are changed\)?)Tj T* (7. How should goals be changed \(be careful about making these changes -\ know why efforts are not )Tj T* (achieving the goals before changing the goals\)? Should any goals be add\ ed or removed? Why?)Tj T* (8. How should goals be established in the future?)Tj /T1_2 1 Tf 0 -2.55714 TD (Process-Based Evaluations)Tj /T1_3 1 Tf ( )Tj /T1_2 1 Tf (\(understanding how your program really works, and its strengths and )Tj 0 -1.2 TD (weaknesses\))Tj /T1_0 1 Tf 0 -1.20001 TD (Process-based evaluations are geared to fully understanding how a progra\ m works -- how does it )Tj 0 -1.2 TD (produce that results that it does. These evaluations are useful if progr\ ams are long-standing and have )Tj T* (changed over the years, employees or customers report a large number of \ complaints about the program, )Tj T* (there appear to be large inefficiencies in delivering program services a\ nd they are also useful for )Tj T* (accurately portraying to outside parties how a program truly operates \(\ e.g., for replication elsewhere\). )Tj 0 -2.20001 TD (There are numerous questions that might be addressed in a process evalua\ tion. These questions can be )Tj 0 -1.2 TD (selected by carefully considering what is important to know about the pr\ ogram. Examples of questions )Tj T* (to ask yourself when designing an evaluation to understand and/or closel\ y examine the processes in your )Tj T* (programs, are:)Tj T* (1. On what basis do employees and/or the customers decide that products \ or services are needed?)Tj 0 -1.20001 TD (2. What is required of employees in order to deliver the product or serv\ ices?)Tj 0 -1.2 TD (3. How are employees trained about how to deliver the product or service\ s?)Tj T* (4. How do customers or clients come into the program?)Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(6 of 15\)10/12/200\ 5 5:05:24 AM)Tj ET EMC endstream endobj 222 0 obj 4588 endobj 223 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_0 1 Tf 14 0 0 14 10 753.19092 Tm (5. What is required of customers or client?)Tj 0 -1.2 TD (6. How do employees select which products or services will be provided t\ o the customer or client?)Tj T* (7. What is the general process that customers or clients go through with\ the product or program?)Tj T* (8. What do customers or clients consider to be strengths of the program?\ )Tj T* (9. What do staff consider to be strengths of the product or program?)Tj T* (10. What typical complaints are heard from employees and/or customers?)Tj T* (11. What do employees and/or customers recommend to improve the product \ or program?)Tj T* (12. On what basis do emplyees and/or the customer decide that the produc\ t or services are no longer )Tj T* (needed?)Tj /T1_1 1 Tf 0 -2.2 TD (Outcomes-Based Evaluation)Tj /T1_2 1 Tf ( )Tj /T1_1 1 Tf (\(identifying benefits to clients\))Tj /T1_0 1 Tf 0 -1.2 TD (Program evaluation with an outcomes focus is increasingly important for \ nonprofits and asked for by )Tj T* (funders.)Tj /T1_2 1 Tf ( )Tj /T1_0 1 Tf (An outcomes-based evaluation facilitates your asking if your organizatio\ n is really doing the )Tj T* (right program activities to bring about the outcomes you believe \(or be\ tter yet, you've verified\) to be )Tj T* (needed by your clients \(rather than just engaging in busy activities wh\ ich seem reasonable to do at the )Tj T* (time\). Outcomes are benefits to clients from participation in the progr\ am. Outcomes are usually in terms )Tj T* (of enhanced learning \(knowledge, perceptions/attitudes or skills\) or c\ onditions, e.g., increased literacy, )Tj T* (self-reliance, etc. Outcomes are often confused with program outputs or \ units of services, e.g., the )Tj T* (number of clients who went through a program.)Tj 0 -2.55714 TD (The )Tj ET 0 0 1 RG 0.7056 w 10 M 0 j 0 J []0 d 35.27 415.67413 m 170.17401 415.67413 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 35.27 417.79092 Tm (United Way of America)Tj 0 0 0 rg ( \(http://www.unitedway.org/outcomes/\) provides an excellent overview o\ f )Tj -1.80499 -1.34286 Td (outcomes-based evaluation, including introduction to outcomes measuremen\ t, a program outcome )Tj 0 -1.2 TD (model, why to measure outcomes, use of program outcome findings by agenc\ ies, eight steps to success )Tj T* (for measuring outcomes, examples of outcomes and outcome indicators for \ various programs and the )Tj T* (resources needed for measuring outcomes. The following information is a \ top-level summary of )Tj T* (information from this site.)Tj 0 -2.55714 TD (To accomplish an outcomes-based evaluation, you should first pilot, or t\ est, this evaluation approach on )Tj 0 -1.2 TD (one or two programs at most \(before doing all programs\).)Tj 0 -2.55714 TD (The general steps to accomplish an outcomes-based evaluation include to:\ )Tj 0 -1.2 TD (1. Identify the major outcomes that you want to examine or verify for th\ e program under evaluation. )Tj T* (You might reflect on your mission \(the overall purpose of your organiza\ tion\) and ask yourself what )Tj T* (impacts you will have on your clients as you work towards your mission. \ For example, if your overall )Tj T* (mission is to provide shelter and resources to abused women, then ask yo\ urself what benefits this will )Tj T* (have on those women if you effectively provide them shelter and other se\ rvices or resources. As a last )Tj T* (resort, you might ask yourself, "What major activities are we doing now?\ " and then for each activity, ask )Tj T* ("Why are we doing that?" The answer to this "Why?" question is usually a\ n outcome. This "last resort" )Tj T* (approach, though, may just end up justifying ineffective activities you \ are doing now, rather than )Tj T* (examining what you should be doing in the first place.)Tj T* (2. Choose the outcomes that you want to examine, prioritize the outcomes\ and, if your time and )Tj T* (resources are limited, pick the top two to four most important outcomes \ to examine for now.)Tj T* (3. For each outcome, specify what observable measures, or indicators, wi\ ll suggest that you're achieving )Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(7 of 15\)10/12/200\ 5 5:05:24 AM)Tj ET EMC endstream endobj 224 0 obj 8793 endobj 225 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /WebCaptureBG BMC /WebCaptureFN <>BDC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.80112 0.80112 0.80112 rg 10 267.90768 m 602 267.90768 l 601 266.90768 l 11 266.90768 l h f 10 267.90768 m 10 36 l 11 36 l 11 266.90768 l h f 0.31348 0.31348 0.31348 rg 602 267.90768 m 602 36 l 601 36 l 601 266.90768 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 13 264.90768 m 105.8 264.90768 l 104.8 263.90768 l 14 263.90768 l h f 13 264.90768 m 13 240.9077 l 14 241.9077 l 14 263.90768 l h f 0.94942 0.9447 0.93529 rg 105.8 264.90768 m 105.8 240.9077 l 104.8 241.9077 l 104.8 263.90768 l h f 13 240.9077 m 105.8 240.9077 l 104.8 241.9077 l 14 241.9077 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 107.8 264.90768 m 264.39999 264.90768 l 263.39999 263.90768 l 108.8 263.90768 l h f 107.8 264.90768 m 107.8 240.9077 l 108.8 241.9077 l 108.8 263.90768 l h f 0.94942 0.9447 0.93529 rg 264.39999 264.90768 m 264.39999 240.9077 l 263.39999 241.9077 l 263.39999 263.90768 l h f 107.8 240.9077 m 264.39999 240.9077 l 263.39999 241.9077 l 108.8 241.9077 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 266.39999 264.90768 m 434.60001 264.90768 l 433.60001 263.90768 l 267.39999 263.90768 l h f 266.39999 264.90768 m 266.39999 240.9077 l 267.39999 241.9077 l 267.39999 263.90768 l h f 0.94942 0.9447 0.93529 rg 434.60001 264.90768 m 434.60001 240.9077 l 433.60001 241.9077 l 433.60001 263.90768 l h f 266.39999 240.9077 m 434.60001 240.9077 l 433.60001 241.9077 l 267.39999 241.9077 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 436.60001 264.90768 m 599 264.90768 l 598 263.90768 l 437.60001 263.90768 l h f 436.60001 264.90768 m 436.60001 240.9077 l 437.60001 241.9077 l 437.60001 263.90768 l h f 0.94942 0.9447 0.93529 rg 599 264.90768 m 599 240.9077 l 598 241.9077 l 598 263.90768 l h f 436.60001 240.9077 m 599 240.9077 l 598 241.9077 l 437.60001 241.9077 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 13 238.9077 m 105.8 238.9077 l 104.8 237.9077 l 14 237.9077 l h f 13 238.9077 m 13 112.84616 l 14 113.84616 l 14 237.9077 l h f 0.94942 0.9447 0.93529 rg 105.8 238.9077 m 105.8 112.84616 l 104.8 113.84616 l 104.8 237.9077 l h f 13 112.84616 m 105.8 112.84616 l 104.8 113.84616 l 14 113.84616 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 107.8 238.9077 m 264.39999 238.9077 l 263.39999 237.9077 l 108.8 237.9077 l h f 107.8 238.9077 m 107.8 112.84616 l 108.8 113.84616 l 108.8 237.9077 l h f 0.94942 0.9447 0.93529 rg 264.39999 238.9077 m 264.39999 112.84616 l 263.39999 113.84616 l 263.39999 237.9077 l h f 107.8 112.84616 m 264.39999 112.84616 l 263.39999 113.84616 l 108.8 113.84616 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 266.39999 238.9077 m 434.60001 238.9077 l 433.60001 237.9077 l 267.39999 237.9077 l h f 266.39999 238.9077 m 266.39999 112.84616 l 267.39999 113.84616 l 267.39999 237.9077 l h f 0.94942 0.9447 0.93529 rg 434.60001 238.9077 m 434.60001 112.84616 l 433.60001 113.84616 l 433.60001 237.9077 l h f 266.39999 112.84616 m 434.60001 112.84616 l 433.60001 113.84616 l 267.39999 113.84616 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 436.60001 238.9077 m 599 238.9077 l 598 237.9077 l 437.60001 237.9077 l h f 436.60001 238.9077 m 436.60001 112.84616 l 437.60001 113.84616 l 437.60001 237.9077 l h f 0.94942 0.9447 0.93529 rg 599 238.9077 m 599 112.84616 l 598 113.84616 l 598 237.9077 l h f 436.60001 112.84616 m 599 112.84616 l 598 113.84616 l 437.60001 113.84616 l h f EMC EMC EMC /Article <>BDC q 0 18 612 756 re W* n 0 0 0 rg BT /T1_0 1 Tf 14 0 0 14 10 753.97552 Tm (that key outcome with your clients. This is often the most important and\ enlightening step in outcomes-)Tj 0 -1.2 TD (based evaluation. However, it is often the most challenging and even con\ fusing step, too, because you're )Tj T* (suddenly going from a rather intangible concept, e.g., increased self-re\ liance, to specific activities, e.g., )Tj T* (supporting clients to get themselves to and from work, staying off drugs\ and alcohol, etc. It helps to have )Tj T* (a "devil's advocate" during this phase of identifying indicators, i.e., \ someone who can question why you )Tj T* (can assume that an outcome was reached because certain associated indica\ tors were present.)Tj T* (4. Specify a "target" goal of clients, i.e., what number or percent of c\ lients you commit to achieving )Tj T* (specific outcomes with, e.g., "increased self-reliance \(an outcome\) fo\ r 70% of adult, African American )Tj T* (women living in the inner city of Minneapolis as evidenced by the follow\ ing measures \(indicators\) ...")Tj T* (5. Identify what information is needed to show these indicators, e.g., y\ ou'll need to know how many )Tj T* (clients in the target group went through the program, how many of them r\ eliably undertook their own )Tj T* (transportation to work and stayed off drugs, etc. If your program is new\ , you may need to evaluate the )Tj T* (process in the program to verify that the program is indeed carried out \ according to your original plans. )Tj T* (\(Michael Patton, prominent researcher, writer and consultant in evaluat\ ion, suggests that the most )Tj T* (important type of evaluation to carry out may be this implementation eva\ luation to verify that your )Tj T* (program ended up to be implemented as you originally planned.\))Tj T* (6. Decide how can that information be efficiently and realistically gath\ ered \(see )Tj ET 0 0 1 RG 0.7056 w 10 M 0 j 0 J []0 d 459.83401 483.05875 m 556.26599 483.05875 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 459.83401 485.17554 Tm (Selecting Which )Tj ET 10 464.25876 m 98.662 464.25876 l S BT /T1_0 1 Tf 14 0 0 14 10 466.37555 Tm (Methods to Use)Tj 0 0 0 rg ( below\). Consider program documentation, observation of program personn\ el and clients )Tj 0 -1.34286 TD (in the program, questionnaires and interviews about clients perceived be\ nefits from the program, case )Tj 0 -1.2 TD (studies of program failures and successes, etc. You may not need all of \ the above. \(see Overview of )Tj ET 10 411.85873 m 191.216 411.85873 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 10 413.97556 Tm (Methods to Collect Information )Tj 0 0 0 rg (below\).)Tj 0 -1.34286 TD (7. Analyze and report the findings \(see )Tj ET 230.83601 393.05875 m 456.34799 393.05875 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 230.83601 395.17554 Tm (Analyzing and Interpreting Information )Tj 0 0 0 rg (below\).)Tj /T1_1 1 Tf 16.15384 0 0 16.15384 10 355.52563 Tm (Overview of Methods to Collect Information )Tj /T1_0 1 Tf 12.92308 0 0 12.92308 320.46075 355.52563 Tm (\(by Carter McNamara, PhD; last revision: Feb 16, )Tj -24.02374 -1.28528 Td (1998\))Tj 14 0 0 14 10 308.48325 Tm (The following table provides an overview of the major methods used for c\ ollecting data during )Tj 0 -1.2 TD (evaluations.)Tj 12.92308 0 0 12.92308 36.24831 249.56206 Tm ( )Tj /T1_1 1 Tf (Method)Tj 8.10966 0 Td (Overall Purpose)Tj 13.70743 0 Td (Advantages)Tj 13.11234 0 Td (Challenges)Tj /T1_0 1 Tf -36.65102 -4.76071 Td (questionnaires, )Tj T* (surveys, )Tj T* (checklists)Tj 7.33571 3 Td (when need to quickly and/or )Tj T* (easily get lots of information )Tj T* (from people in a non )Tj T* (threatening way)Tj 12.27261 5.39999 Td (-can complete anonymously)Tj T* (-inexpensive to administer)Tj T* (-easy to compare and analyze)Tj T* (-administer to many people)Tj T* (-can get lots of data)Tj T* (-many sample questionnaires )Tj T* (already exist)Tj 13.17024 7.8 Td (-might not get careful )Tj T* (feedback)Tj T* (-wording can bias client's )Tj T* (responses)Tj T* (-are impersonal)Tj T* (-in surveys, may need )Tj T* (sampling expert)Tj T* (- doesn't get full story)Tj ET EMC /Artifact <>BDC Q 0 0 0 rg BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(8 of 15\)10/12/200\ 5 5:05:24 AM)Tj ET EMC endstream endobj 226 0 obj 13850 endobj 227 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /WebCaptureBG BMC /WebCaptureFN <>BDC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.80112 0.80112 0.80112 rg 10 766 m 10 194.44615 l 11 195.44615 l 11 766 l h f 0.31348 0.31348 0.31348 rg 602 766 m 602 194.44615 l 601 195.44615 l 601 766 l h f 10 194.44615 m 602 194.44615 l 601 195.44615 l 11 195.44615 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 13 764 m 105.8 764 l 104.8 763 l 14 763 l h f 13 764 m 13 656 l 14 657 l 14 763 l h f 0.94942 0.9447 0.93529 rg 105.8 764 m 105.8 656 l 104.8 657 l 104.8 763 l h f 13 656 m 105.8 656 l 104.8 657 l 14 657 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 107.8 764 m 264.39999 764 l 263.39999 763 l 108.8 763 l h f 107.8 764 m 107.8 656 l 108.8 657 l 108.8 763 l h f 0.94942 0.9447 0.93529 rg 264.39999 764 m 264.39999 656 l 263.39999 657 l 263.39999 763 l h f 107.8 656 m 264.39999 656 l 263.39999 657 l 108.8 657 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 266.39999 764 m 434.60001 764 l 433.60001 763 l 267.39999 763 l h f 266.39999 764 m 266.39999 656 l 267.39999 657 l 267.39999 763 l h f 0.94942 0.9447 0.93529 rg 434.60001 764 m 434.60001 656 l 433.60001 657 l 433.60001 763 l h f 266.39999 656 m 434.60001 656 l 433.60001 657 l 267.39999 657 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 436.60001 764 m 599 764 l 598 763 l 437.60001 763 l h f 436.60001 764 m 436.60001 656 l 437.60001 657 l 437.60001 763 l h f 0.94942 0.9447 0.93529 rg 599 764 m 599 656 l 598 657 l 598 763 l h f 436.60001 656 m 599 656 l 598 657 l 437.60001 657 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 13 654 m 105.8 654 l 104.8 653 l 14 653 l h f 13 654 m 13 538 l 14 539 l 14 653 l h f 0.94942 0.9447 0.93529 rg 105.8 654 m 105.8 538 l 104.8 539 l 104.8 653 l h f 13 538 m 105.8 538 l 104.8 539 l 14 539 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 107.8 654 m 264.39999 654 l 263.39999 653 l 108.8 653 l h f 107.8 654 m 107.8 538 l 108.8 539 l 108.8 653 l h f 0.94942 0.9447 0.93529 rg 264.39999 654 m 264.39999 538 l 263.39999 539 l 263.39999 653 l h f 107.8 538 m 264.39999 538 l 263.39999 539 l 108.8 539 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 266.39999 654 m 434.60001 654 l 433.60001 653 l 267.39999 653 l h f 266.39999 654 m 266.39999 538 l 267.39999 539 l 267.39999 653 l h f 0.94942 0.9447 0.93529 rg 434.60001 654 m 434.60001 538 l 433.60001 539 l 433.60001 653 l h f 266.39999 538 m 434.60001 538 l 433.60001 539 l 267.39999 539 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 436.60001 654 m 599 654 l 598 653 l 437.60001 653 l h f 436.60001 654 m 436.60001 538 l 437.60001 539 l 437.60001 653 l h f 0.94942 0.9447 0.93529 rg 599 654 m 599 538 l 598 539 l 598 653 l h f 436.60001 538 m 599 538 l 598 539 l 437.60001 539 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 13 536 m 105.8 536 l 104.8 535 l 14 535 l h f 13 536 m 13 425.44614 l 14 426.44614 l 14 535 l h f 0.94942 0.9447 0.93529 rg 105.8 536 m 105.8 425.44614 l 104.8 426.44614 l 104.8 535 l h f 13 425.44614 m 105.8 425.44614 l 104.8 426.44614 l 14 426.44614 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 107.8 536 m 264.39999 536 l 263.39999 535 l 108.8 535 l h f 107.8 536 m 107.8 425.44614 l 108.8 426.44614 l 108.8 535 l h f 0.94942 0.9447 0.93529 rg 264.39999 536 m 264.39999 425.44614 l 263.39999 426.44614 l 263.39999 535 l h f 107.8 425.44614 m 264.39999 425.44614 l 263.39999 426.44614 l 108.8 426.44614 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 266.39999 536 m 434.60001 536 l 433.60001 535 l 267.39999 535 l h f 266.39999 536 m 266.39999 425.44614 l 267.39999 426.44614 l 267.39999 535 l h f 0.94942 0.9447 0.93529 rg 434.60001 536 m 434.60001 425.44614 l 433.60001 426.44614 l 433.60001 535 l h f 266.39999 425.44614 m 434.60001 425.44614 l 433.60001 426.44614 l 267.39999 426.44614 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 436.60001 536 m 599 536 l 598 535 l 437.60001 535 l h f 436.60001 536 m 436.60001 425.44614 l 437.60001 426.44614 l 437.60001 535 l h f 0.94942 0.9447 0.93529 rg 599 536 m 599 425.44614 l 598 426.44614 l 598 535 l h f 436.60001 425.44614 m 599 425.44614 l 598 426.44614 l 437.60001 426.44614 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 13 423.44614 m 105.8 423.44614 l 104.8 422.44614 l 14 422.44614 l h f 13 423.44614 m 13 299.44614 l 14 300.44614 l 14 422.44614 l h f 0.94942 0.9447 0.93529 rg 105.8 423.44614 m 105.8 299.44614 l 104.8 300.44614 l 104.8 422.44614 l h f 13 299.44614 m 105.8 299.44614 l 104.8 300.44614 l 14 300.44614 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 107.8 423.44614 m 264.39999 423.44614 l 263.39999 422.44614 l 108.8 422.44614 l h f 107.8 423.44614 m 107.8 299.44614 l 108.8 300.44614 l 108.8 422.44614 l h f 0.94942 0.9447 0.93529 rg 264.39999 423.44614 m 264.39999 299.44614 l 263.39999 300.44614 l 263.39999 422.44614 l h f 107.8 299.44614 m 264.39999 299.44614 l 263.39999 300.44614 l 108.8 300.44614 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 266.39999 423.44614 m 434.60001 423.44614 l 433.60001 422.44614 l 267.39999 422.44614 l h f 266.39999 423.44614 m 266.39999 299.44614 l 267.39999 300.44614 l 267.39999 422.44614 l h f 0.94942 0.9447 0.93529 rg 434.60001 423.44614 m 434.60001 299.44614 l 433.60001 300.44614 l 433.60001 422.44614 l h f 266.39999 299.44614 m 434.60001 299.44614 l 433.60001 300.44614 l 267.39999 300.44614 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 436.60001 423.44614 m 599 423.44614 l 598 422.44614 l 437.60001 422.44614 l h f 436.60001 423.44614 m 436.60001 299.44614 l 437.60001 300.44614 l 437.60001 422.44614 l h f 0.94942 0.9447 0.93529 rg 599 423.44614 m 599 299.44614 l 598 300.44614 l 598 422.44614 l h f 436.60001 299.44614 m 599 299.44614 l 598 300.44614 l 437.60001 300.44614 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 13 297.44614 m 105.8 297.44614 l 104.8 296.44614 l 14 296.44614 l h f 13 297.44614 m 13 197.44615 l 14 198.44615 l 14 296.44614 l h f 0.94942 0.9447 0.93529 rg 105.8 297.44614 m 105.8 197.44615 l 104.8 198.44615 l 104.8 296.44614 l h f 13 197.44615 m 105.8 197.44615 l 104.8 198.44615 l 14 198.44615 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 107.8 297.44614 m 264.39999 297.44614 l 263.39999 296.44614 l 108.8 296.44614 l h f 107.8 297.44614 m 107.8 197.44615 l 108.8 198.44615 l 108.8 296.44614 l h f 0.94942 0.9447 0.93529 rg 264.39999 297.44614 m 264.39999 197.44615 l 263.39999 198.44615 l 263.39999 296.44614 l h f 107.8 197.44615 m 264.39999 197.44615 l 263.39999 198.44615 l 108.8 198.44615 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 266.39999 297.44614 m 434.60001 297.44614 l 433.60001 296.44614 l 267.39999 296.44614 l h f 266.39999 297.44614 m 266.39999 197.44615 l 267.39999 198.44615 l 267.39999 296.44614 l h f 0.94942 0.9447 0.93529 rg 434.60001 297.44614 m 434.60001 197.44615 l 433.60001 198.44615 l 433.60001 296.44614 l h f 266.39999 197.44615 m 434.60001 197.44615 l 433.60001 198.44615 l 267.39999 198.44615 l h f EMC q 0 18 612 756 re W* n /Artifact <>BDC Q 0.45726 0.44862 0.43137 rg 436.60001 297.44614 m 599 297.44614 l 598 296.44614 l 437.60001 296.44614 l h f 436.60001 297.44614 m 436.60001 197.44615 l 437.60001 198.44615 l 437.60001 296.44614 l h f 0.94942 0.9447 0.93529 rg 599 297.44614 m 599 197.44615 l 598 198.44615 l 598 296.44614 l h f 436.60001 197.44615 m 599 197.44615 l 598 198.44615 l 437.60001 198.44615 l h f EMC EMC EMC /Article <>BDC q 0 18 612 756 re W* n 0 0 0 rg BT /T1_0 1 Tf 12.92308 0 0 12.92308 14 706.65436 Tm (interviews)Tj 7.33571 2.39999 Td (when want to fully )Tj 0 -1.2 TD (understand someone's )Tj T* (impressions or experiences, )Tj T* (or learn more about their )Tj T* (answers to questionnaires)Tj 12.27261 4.77843 Td (-)Tj 14 0 0 14 271.70337 737.39093 Tm (g)Tj 12.92308 0 0 12.92308 278.70337 737.39093 Tm (et full range and depth of )Tj -0.87466 -1.22842 Td (information)Tj T* (-develops relationship with )Tj T* (client)Tj 0 -1.20001 TD (-can be flexible with client)Tj 14 0 0 14 437.60001 745.14478 Tm (-)Tj 12.92308 0 0 12.92308 442.26199 745.14478 Tm (can take much time)Tj -0.36075 -1.22842 Td (-can be hard to analyze and )Tj 0 -1.2 TD (compare)Tj T* (-can be costly)Tj T* (-interviewer can bias client's )Tj 0 -1.20001 TD (responses)Tj -32.77856 -5.17142 Td (documentation )Tj 0 -1.2 TD (review)Tj 7.33571 3.60001 Td (when want impression of )Tj T* (how program operates )Tj T* (without interrupting the )Tj T* (program; is from review of )Tj T* (applications, finances, )Tj T* (memos, minutes, etc.)Tj 12.27261 6 Td (-get comprehensive and )Tj T* (historical information)Tj T* (-doesn't interrupt program or )Tj T* (client's routine in program)Tj T* (-information already exists)Tj T* (-few biases about information)Tj 13.17023 6.60001 Td (-often takes much time)Tj T* (-info may be incomplete)Tj T* (-need to be quite clear about )Tj T* (what looking for)Tj T* (-not flexible means to get )Tj T* (data; data restricted to what )Tj T* (already exists)Tj 14 0 0 14 14 477.09863 Tm ( )Tj 12.92308 0 0 12.92308 17.5 477.09863 Tm (observation)Tj 7.06488 1.82156 Td (to gather accurate )Tj T* (information about how a )Tj T* (program actually operates, )Tj T* (particularly about processes)Tj 12.27261 3.60001 Td (-view operations of a program )Tj T* (as they are actually occurring)Tj T* (-can adapt to events as they )Tj T* (occur)Tj 13.17024 5.39999 Td (-can be difficult to interpret )Tj T* (seen behaviors)Tj T* (-can be complex to categorize )Tj T* (observations)Tj T* (-can influence behaviors of )Tj T* (program participants)Tj T* (-can be expensive)Tj 14 0 0 14 14 357.82172 Tm (f)Tj 12.92308 0 0 12.92308 18.662 357.82172 Tm (ocus groups)Tj 6.97496 3.62157 Td (explore a topic in depth )Tj T* (through group discussion, e.)Tj T* (g., about reactions to an )Tj T* (experience or suggestion, )Tj T* (understanding common )Tj T* (complaints, etc.; useful in )Tj T* (evaluation and marketing)Tj 14 0 0 14 267.39999 404.34479 Tm (-)Tj 12.92308 0 0 12.92308 272.06201 404.34479 Tm (quickly and reliably get )Tj -0.36075 -1.22842 Td (common impressions )Tj T* (-can be efficient way to get )Tj T* (much range and depth of )Tj T* (information in short time)Tj T* (- can convey key information )Tj T* (about programs)Tj 13.17024 6.64999 Td (-can be hard to analyze )Tj T* (responses)Tj T* (-need good facilitator for )Tj T* (safety and closure)Tj T* (-difficult to schedule 6-8 )Tj T* (people together)Tj -32.77856 -5.82143 Td (case studies)Tj 7.33571 3 Td (to fully understand or depict )Tj T* (client's experiences in a )Tj T* (program, and conduct )Tj T* (comprehensive examination )Tj T* (through cross comparison of )Tj T* (cases)Tj 14 0 0 14 267.39999 274.8371 Tm (-)Tj 12.92308 0 0 12.92308 272.06201 274.8371 Tm (fully depicts client's )Tj -0.36075 -1.22842 Td (experience in program input, )Tj T* (process and results)Tj T* (-powerful means to portray )Tj T* (program to outsiders)Tj 13.17024 5.45 Td (-usually quite time consuming )Tj T* (to collect, organize and )Tj T* (describe )Tj T* (-represents depth of )Tj T* (information, rather than )Tj T* (breadth)Tj /T1_1 1 Tf 14 0 0 14 10 163.42171 Tm (Also see)Tj ET 0 0 1 RG 0.7056 w 10 M 0 j 0 J []0 d 10 144.50491 m 127.02602 144.50491 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 10 146.62169 Tm (Appreciative Inquiry)Tj ET 10 125.70491 m 92.82401 125.70491 l S BT /T1_0 1 Tf 14 0 0 14 10 127.82172 Tm (Survey Design)Tj 0 0 0 rg /T1_2 1 Tf 0 -2.7 TD (Ethics: Informed Consent from Program Participants)Tj /T1_0 1 Tf 0 -1.2 TD (Note that if you plan to include in your evaluation, the focus and repor\ ting on personal information )Tj T* (about customers or clients participating in the evaluation, then you sho\ uld first gain their consent to do )Tj T* (so. They should understand what you're doing with them in the evaluation\ and how any information )Tj ET EMC /Artifact <>BDC Q 0 0 0 rg BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(9 of 15\)10/12/200\ 5 5:05:24 AM)Tj ET EMC endstream endobj 228 0 obj 4345 endobj 229 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_0 1 Tf 14 0 0 14 10 752.82172 Tm (associated with them will be reported. You should clearly convey terms o\ f confidentiality regarding )Tj 0 -1.2 TD (access to evaluation results. They should have the right to participate \ or not. Have participants review )Tj T* (and sign an informed consent form. See the )Tj ET 0 0 1 RG 0.7056 w 10 M 0 j 0 J []0 d 256.90399 717.10486 m 429.53799 717.10486 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 256.90399 719.22168 Tm (sample informed-consent form)Tj 0 0 0 rg (.)Tj /T1_1 1 Tf -17.636 -2.7 Td (How to Apply Certain Methods)Tj ET 10 662.50488 m 208.32401 662.50488 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 10 664.6217 Tm (Purposes and Formats of Questions)Tj ET 10 643.7049 m 162.81 643.7049 l S BT /T1_0 1 Tf 14 0 0 14 10 645.82172 Tm (Developing Questionnaires)Tj ET 10 624.90491 m 137.93201 624.90491 l S BT /T1_0 1 Tf 14 0 0 14 10 627.02167 Tm (Conducting Interviews)Tj ET 10 606.10486 m 157 606.10486 l S BT /T1_0 1 Tf 14 0 0 14 10 608.22168 Tm (Conducting Focus Groups)Tj ET 10 587.30493 m 150.76999 587.30493 l S BT /T1_0 1 Tf 14 0 0 14 10 589.42169 Tm (Developing Case Studies)Tj 0 0 0 rg /T1_2 1 Tf 16.15384 0 0 16.15384 10 549.77179 Tm (Selecting Which Methods to Use )Tj /T1_0 1 Tf 14 0 0 14 236.59 549.77179 Tm (\(by Carter McNamara, PhD; last revision: Feb 16, 1998\))Tj /T1_1 1 Tf -16.185 -1.25249 Td (Overall Goal in Selecting Methods:)Tj /T1_0 1 Tf T* (The overall goal in selecting evaluation method\(s\) is to get the most \ useful information to key decision )Tj T* (makers in the most cost-effective and realistic fashion. Consider the fo\ llowing questions:)Tj T* (1. What information is needed to make current decisions about a product \ or program?)Tj T* (2. Of this information, how much can be collected and analyzed in a low-\ cost and practical manner, e.g., )Tj T* (using questionnaires, surveys and checklists?)Tj T* (3. How accurate will the information be \(reference the above table for \ disadvantages of methods\)?)Tj T* (4. Will the methods get all of the needed information?)Tj T* (5. What additional methods should and could be used if additional inform\ ation is needed?)Tj T* (6. Will the information appear as credible to decision makers, e.g., to \ funders or top management?)Tj T* (7. Will the nature of the audience conform to the methods, e.g., will th\ ey fill out questionnaires )Tj T* (carefully, engage in interviews or focus groups, let you examine their d\ ocumentations, etc.?)Tj T* (8. Who can administer the methods now or is training required?)Tj T* (9. How can the information be analyzed?)Tj 0 -2.55714 TD (Note that, ideally, the evaluator uses a combination of methods, for exa\ mple, a questionnaire to quickly )Tj 0 -1.2 TD (collect a great deal of information from a lot of people, and then inter\ views to get more in-depth )Tj T* (information from certain respondents to the questionnaires. Perhaps case\ studies could then be used for )Tj T* (more in-depth analysis of unique and notable cases, e.g., those who bene\ fited or not from the program, )Tj T* (those who quit the program, etc.)Tj /T1_1 1 Tf 0 -2.55714 TD (Four Levels of Evaluation:)Tj /T1_0 1 Tf 0 -1.2 TD (There are four levels of evaluation information that can be gathered fro\ m clients, including getting their:)Tj T* (1. reactions and feelings \(feelings are often poor indicators that your\ service made lasting impact\))Tj T* (2. learning \(enhanced attitudes, perceptions or knowledge\))Tj T* (3. changes in skills \(applied the learning to enhance behaviors\))Tj T* (4. effectiveness \(improved performance because of enhanced behaviors\))Tj 0 -2.55714 TD (Usually, the farther your evaluation information gets down the list, the\ more useful is your evaluation. )Tj 0 -1.2 TD (Unfortunately, it is quite difficult to reliably get information about e\ ffectiveness. Still, information about )Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(10 of 15\)10/12/20\ 05 5:05:24 AM)Tj ET EMC endstream endobj 230 0 obj 4058 endobj 231 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_0 1 Tf 14 0 0 14 10 751.63708 Tm (learning and skills is quite useful.)Tj /T1_1 1 Tf 16.15384 0 0 16.15384 10 713.98718 Tm (Analyzing and Interpreting Information )Tj /T1_0 1 Tf 14 0 0 14 291.80383 713.98718 Tm (\(Carter McNamara, PhD; last revision: Feb 16, 1998\))Tj -20.12885 -1.25249 Td (Analyzing quantitative and qualitative data is often the topic of advanc\ ed research and evaluation )Tj 0 -1.2 TD (methods. There are certain basics which can help to make sense of reams \ of data.)Tj /T1_2 1 Tf 0 -2.55714 TD (Always start with your evaluation goals:)Tj /T1_0 1 Tf 0 -1.2 TD (When analyzing data \(whether from questionnaires, interviews, focus gro\ ups, or whatever\), always start )Tj T* (from review of your evaluation goals, i.e., the reason you undertook the\ evaluation in the first place. )Tj T* (This will help you organize your data and focus your analysis. For examp\ le, if you wanted to improve )Tj T* (your program by identifying its strengths and weaknesses, you can organi\ ze data into program strengths, )Tj T* (weaknesses and suggestions to improve the program. If you wanted to full\ y understand how your )Tj T* (program works, you could organize data in the chronological order in whi\ ch clients go through your )Tj T* (program. If you are conducting an outcomes-based evaluation, you can cat\ egorize data according to the )Tj T* (indicators for each outcome.)Tj /T1_2 1 Tf 0 -2.55714 TD (Basic analysis of "quantitative" information )Tj /T1_3 1 Tf (\(for information other than commentary, e.g., ratings, )Tj 0 -1.2 TD (rankings, yes's, no's, etc.\):)Tj /T1_0 1 Tf T* (1. Make copies of your data and store the master copy away. Use the copy\ for making edits, cutting and )Tj 0 -1.20001 TD (pasting, etc.)Tj 0 -1.2 TD (2. Tabulate the information, i.e., add up the number of ratings, ranking\ s, yes's, no's for each question. )Tj T* (3. For ratings and rankings, consider computing a mean, or average, for \ each question. For example, )Tj T* ("For question #1, the average ranking was 2.4". This is more meaningful \ than indicating, e.g., how many )Tj T* (respondents ranked 1, 2, or 3. )Tj 0 -1.20001 TD (4. Consider conveying the range of answers, e.g., 20 people ranked "1", \ 30 ranked "2", and 20 people )Tj 0 -1.2 TD (ranked "3".)Tj /T1_2 1 Tf 0 -2.55714 TD (Basic analysis of "qualitative" information )Tj /T1_3 1 Tf (\(respondents' verbal answers in interviews, focus groups, )Tj 0 -1.2 TD (or written commentary on questionnaires\):)Tj /T1_0 1 Tf T* (1. Read through all the data.)Tj T* (2. Organize comments into similar categories, e.g., concerns, suggestion\ s, strengths, weaknesses, similar )Tj T* (experiences, program inputs, recommendations, outputs, outcome indicator\ s, etc.)Tj T* (3. Label the categories or themes, e.g., concerns, suggestions, etc.)Tj T* (4. Attempt to identify patterns, or associations and causal relationship\ s in the themes, e.g., all people )Tj T* (who attended programs in the evening had similar concerns, most people c\ ame from the same )Tj T* (geographic area, most people were in the same salary range, what process\ es or events respondents )Tj T* (experience during the program, etc.)Tj T* (4. Keep all commentary for several years after completion in case needed\ for future reference.)Tj /T1_2 1 Tf 0 -2.55714 TD (Interpreting Information:)Tj /T1_0 1 Tf 0 -1.2 TD (1. Attempt to put the information in perspective, e.g., compare results \ to what you expected, promised )Tj T* (results; management or program staff; any common standards for your serv\ ices; original program goals )Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(11 of 15\)10/12/20\ 05 5:05:24 AM)Tj ET EMC endstream endobj 232 0 obj 4659 endobj 233 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_0 1 Tf 14 0 0 14 10 753.97552 Tm (\(especially if you're conducting a program evaluation\); indications of\ accomplishing outcomes )Tj 0 -1.2 TD (\(especially if you're conducting an outcomes evaluation\); description \ of the program's experiences, )Tj T* (strengths, weaknesses, etc. \(especially if you're conducting a process \ evaluation\).)Tj T* (2. Consider recommendations to help program staff improve the program, c\ onclusions about program )Tj T* (operations or meeting goals, etc.)Tj T* (3. Record conclusions and recommendations in a report document, and asso\ ciate interpretations to )Tj T* (justify your conclusions or recommendations.)Tj /T1_1 1 Tf 16.15384 0 0 16.15384 10 615.52563 Tm (Reporting Evaluation Results )Tj /T1_0 1 Tf 14 0 0 14 217.77077 615.52563 Tm (\(by Carter McNamara, PhD; last revision: Feb 16, 1998\))Tj -14.84077 -1.36923 Td (1.)Tj /T1_1 1 Tf 16.15384 0 0 16.15384 20.5 596.35638 Tm ( )Tj /T1_0 1 Tf 14 0 0 14 24.53847 596.35638 Tm (The level and scope of content depends on to whom the report is intended\ , e.g., to bankers, funders, )Tj -1.03847 -1.23709 Td (employees, customers, clients, the public, etc.)Tj T* (2. Be sure employees have a chance to carefully review and discuss the r\ eport. Translate )Tj T* (recommendations to action plans, including who is going to do what about\ the program and by when.)Tj T* (3. Bankers or funders will likely require a report that includes an exec\ utive summary \(this is a summary )Tj T* (of conclusions and recommendations, not a listing of what sections of in\ formation are in the report -- )Tj T* (that's a table of contents\); description of theorganization and the pro\ gram under evaluation; explanation )Tj T* (of the evaluation goals, methods, and analysis procedures; listing of co\ nclusions and recommendations; )Tj T* (and any relevant attachments, e.g., inclusion of evaluation questionnair\ es, interview guides, etc. The )Tj T* (banker or funder may want the report to be delivered as a presentation, \ accompanied by an overview of )Tj T* (the report. Or, the banker or funder may want to review the report alone\ .)Tj T* (4. Be sure to record the evaluation plans and activities in an evaluatio\ n plan which can be referenced )Tj T* (when a similar program evaluation is needed in the future.)Tj /T1_2 1 Tf 0 -2.55714 TD (Contents of an Evaluation Report -- Example)Tj /T1_0 1 Tf 0 -1.2 TD (An example of evaluation report contents is included later on below in t\ his document. Click )Tj ET 0 0 1 RG 0.7056 w 10 M 0 j 0 J []0 d 527.94403 339.52029 m 596.39001 339.52029 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 527.94403 341.63708 Tm (Contents of )Tj ET 10 320.72028 m 115.77 320.72028 l S BT /T1_0 1 Tf 14 0 0 14 10 322.8371 Tm (an Evaluation Plan)Tj 0 0 0 rg ( but, don't forget to look at the next section "Who Should Carry Out the\ Evaluation".)Tj /T1_1 1 Tf 16.15384 0 0 16.15384 10 283.18716 Tm (Who Should Carry Out the Evaluation? )Tj /T1_0 1 Tf 14 0 0 14 290.04306 283.18716 Tm (\(by Carter McNamara, PhD; last revision: Feb 16, )Tj -20.00308 -1.25249 Td (1998\))Tj T* (Ideally, management decides what the evaluation goals should be. Then an\ evaluation expert helps the )Tj T* (organization to determine what the evaluation methods should be, and how\ the resulting data will be )Tj T* (analyzed and reported back to the organization. Most organizations do no\ t have the resources to carry )Tj T* (out the ideal evaluation.)Tj 0 -2.55714 TD (Still, they can do the 20% of effort needed to generate 80% of what they\ need to know to make a )Tj 0 -1.2 TD (decision about a program. If they can afford any outside help at all, it\ should be for identifying the )Tj T* (appropriate evaluation methods and how the data can be collected. The or\ ganization might find a less )Tj T* (expensive resource to apply the methods, e.g., conduct interviews, send \ out and analyze results of )Tj T* (questionnaires, etc.)Tj 0 -2.55714 TD (If no outside help can be obtained, the organization can still learn a g\ reat deal by applying the methods )Tj 0 -1.2 TD (and analyzing results themselves. However, there is a strong chance that\ data about the strengths and )Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(12 of 15\)10/12/20\ 05 5:05:24 AM)Tj ET EMC endstream endobj 234 0 obj 4011 endobj 235 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_0 1 Tf 14 0 0 14 10 753.97552 Tm (weaknesses of a program will not be interpreted fairly if the data are a\ nalyzed by the people responsible )Tj 0 -1.2 TD (for ensuring the program is a good one. Program managers will be "polici\ ng" themselves. This caution is )Tj T* (not to fault program managers, but to recognize the strong biases inhere\ nt in trying to objectively look at )Tj T* (and publicly \(at least within the organization\) report about their pro\ grams. Therefore, if at all possible, )Tj T* (have someone other than the program managers look at and determine evalu\ ation results.)Tj /T1_1 1 Tf 16.15384 0 0 16.15384 10 649.12561 Tm (Contents of an Evaluation Plan)Tj 14 0 0 14 225.41153 649.12561 Tm ( )Tj /T1_0 1 Tf (\(by Carter McNamara, PhD\))Tj -15.38654 -1.25249 Td (Develop an evaluation plan to ensure your program evaluations are carrie\ d out efficiently in the future. )Tj T* (Note that bankers or funders may want or benefit from a copy of this pla\ n.)Tj 0 -2.55714 TD (Ensure your evaluation plan is documented so you can regularly and effic\ iently carry out your )Tj 0 -1.2 TD (evaluation activities. Record enough information in the plan so that som\ eone outside of the organization )Tj T* (can understand what you're evaluating and how. Consider the following fo\ rmat for your report:)Tj T* (1. Title Page \(name of the organization that is being, or has a product\ /service/program that is being, )Tj T* (evaluated; date\))Tj T* (2. Table of Contents)Tj T* (3. Executive Summary \(one-page, concise overview of findings and recomm\ endations\))Tj T* (4. Purpose of the Report \(what type of evaluation\(s\) was conducted, w\ hat decisions are being aided by )Tj T* (the findings of the evaluation, who is making the decision, etc.\))Tj T* (5. Background About Organization and Product/Service/Program that is bei\ ng evaluated)Tj T* (a\) Organization Description/History)Tj T* (b\) Product/Service/Program Description \(that is being evaluated\))Tj T* (i\) Problem Statement \(in the case of nonprofits, description of the co\ mmunity need that is being met by )Tj T* (the product/service/program\))Tj T* (ii\) Overall Goal\(s\) of Product/Service/Program )Tj T* (iii\) Outcomes \(or client/customer impacts\) and Performance Measures \(\ that can be measured as )Tj T* (indicators toward the outcomes\))Tj T* (iv\) Activities/Technologies of the Product/Service/Program \(general de\ scription of how the product/)Tj T* (service/program is developed and delivered\))Tj T* (v\) Staffing \(description of the number of personnel and roles in the o\ rganization that are relevant to )Tj T* (developing and delivering the product/service/program\))Tj T* (6\) Overall Evaluation Goals \(eg, what questions are being answered by \ the evaluation\))Tj T* (7\) Methodology )Tj T* (a\) Types of data/information that were collected)Tj T* (b\) How data/information were collected \(what instruments were used, et\ c.\))Tj T* (c\) How data/information were analyzed)Tj T* (d\) Limitations of the evaluation \(eg, cautions about findings/conclusi\ ons and how to use the findings/)Tj T* (conclusions, etc.\))Tj T* (8\) Interpretations and Conclusions \(from analysis of the data/informat\ ion\))Tj T* (9\) Recommendations \(regarding the decisions that must be made about th\ e product/service/program\))Tj T* (Appendices: content of the appendices depends on the goals of the evalua\ tion report, eg.:)Tj T* (a\) Instruments used to collect data/information)Tj T* (b\) Data, eg, in tabular format, etc.)Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(13 of 15\)10/12/20\ 05 5:05:24 AM)Tj ET EMC endstream endobj 236 0 obj 4818 endobj 237 0 obj<>stream /Artifact <>BDC 0 0 0 rg 0 i BT /T1_0 1 Tf 0 Tc 0 Tw 0 Ts 100 Tz 0 Tr 9 0 0 9 18 780.17 Tm (Basic Guide to Program Evaluation)Tj ET EMC /Article <>BDC q 0 18 612 756 re W* n BT /T1_0 1 Tf 14 0 0 14 10 753.97552 Tm (c\) Testimonials, comments made by users of the product/service/program)Tj 0 -1.2 TD (d\) Case studies of users of the product/service/program)Tj T* (e\) Any related literature)Tj /T1_1 1 Tf 16.15384 0 0 16.15384 10 682.72565 Tm (Pitfalls to Avoid )Tj /T1_0 1 Tf 14 0 0 14 125.32231 682.72565 Tm (\(by Carter McNamara, PhD; last revision: Feb 16, 1998\))Tj -8.2373 -1.25249 Td (1. Don't balk at evaluation because it seems far too "scientific." It's \ not. Usually the first 20% of effort )Tj T* (will generate the first 80% of the plan, and this is far better than not\ hing.)Tj T* (2. There is no "perfect" evaluation design. Don't worry about the plan b\ eing perfect. It's far more )Tj T* (important to do something, than to wait until every last detail has been\ tested.)Tj T* (3. Work hard to include some interviews in your evaluation methods. Ques\ tionnaires don't capture "the )Tj T* (story," and the story is usually the most powerful depiction of the bene\ fits of your services.)Tj T* (4. Don't interview just the successes. You'll learn a great deal about t\ he program by understanding its )Tj T* (failures, dropouts, etc.)Tj T* (5. Don't throw away evaluation results once a report has been generated.\ Results don't take up much )Tj T* (room, and they can provide precious information later when trying to und\ erstand changes in the program.)Tj /T1_1 1 Tf 16.15384 0 0 16.15384 10 476.341 Tm (Additional Resources)Tj ET 0 0 1 RG 0.7056 w 10 M 0 j 0 J []0 d 10 456.68951 m 342.06598 456.68951 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 10 458.8063 Tm (Additional Resources, Guides for many types of evaluation)Tj 0 0 0 rg /T1_1 1 Tf 16.15384 0 0 16.15384 10 419.1564 Tm (Related Library Links)Tj ET 10 399.50491 m 300.51401 399.50491 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 10 401.6217 Tm (Controlling / Coordinating Implementation of Plans)Tj ET 10 380.7049 m 533.01196 380.7049 l S BT /T1_0 1 Tf 14 0 0 14 10 382.82172 Tm (Evaluation Activities in Organizations \(many kinds of evaluations and d\ ocuments\), including)Tj 0 0 0 rg 0 -1.34286 TD (- - - )Tj ET 34.48599 361.90491 m 351.38998 361.90491 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 34.48599 364.0217 Tm (User-Friendly Handbook for Mixed Method Evaluations)Tj 0 0 0 rg -1.74899 -1.34286 Td (- - - )Tj ET 34.48599 343.10489 m 260.194 343.10489 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 34.48599 345.22171 Tm (Program Manager's Guide to Evaluation)Tj 0 0 0 rg -1.74899 -1.34286 Td (- - - )Tj ET 34.48599 324.3049 m 312.90399 324.3049 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 34.48599 326.42169 Tm (Analytical Methods in Maternal and Child Health)Tj 0 0 0 rg -1.74899 -1.34286 Td (- - - )Tj ET 34.48599 305.50491 m 575.71198 305.50491 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 34.48599 307.6217 Tm (General Accounting Office Policy and Guidance Publications \(many free e\ valuation pulications\))Tj ET 10 286.7049 m 150.75601 286.7049 l S BT /T1_0 1 Tf 14 0 0 14 10 288.82172 Tm (Organizing \(many kinds\))Tj ET 10 267.90491 m 137.554 267.90491 l S BT /T1_0 1 Tf 14 0 0 14 10 270.0217 Tm (Planning \(many kinds\))Tj ET 10 249.10489 m 128.59399 249.10489 l S BT /T1_0 1 Tf 14 0 0 14 10 251.22171 Tm (Quality Management)Tj ET 10 230.30492 m 481.64603 230.30492 l S BT /T1_0 1 Tf 14 0 0 14 10 232.42171 Tm (Research Methods \(Basic Business, e.g., focus groups, surveys, question\ naires, etc.\))Tj 0 0 0 rg /T1_1 1 Tf 16.15384 0 0 16.15384 10 192.77179 Tm (On-Line Discussion Groups, Newsletters, etc.)Tj /T1_0 1 Tf 14 0 0 14 10 175.23709 Tm (There are a large number of on-line discussion groups, newsletters \(e-z\ ines\), etc. in the overall areas of )Tj 0 -1.2 TD (management, business and organization development. Participants, subscri\ bers, etc., can get answers to )Tj T* (their questions and learn a lot just by posing the questions to the grou\ ps, sharing insights about their )Tj T* (experiences, etc. Join some groups and sign up for some newsletters!)Tj ET 10 105.92029 m 267.37601 105.92029 l S 0 0 1 rg BT /T1_0 1 Tf 14 0 0 14 10 108.03709 Tm (References to major egroups, newsletters, etc.)Tj 0 0 0 rg /T1_1 1 Tf 16.15384 0 0 16.15384 10 68.38718 Tm (You Can See Useful Books On This Library Topic Right Now!)Tj /T1_0 1 Tf 14 0 0 14 10 50.85248 Tm (Just enter the name of this Library topic in the window below and then c\ lick on "go"!)Tj ET EMC /Artifact <>BDC Q BT /T1_0 1 Tf 9 0 0 9 18 7.17 Tm (http://www.managementhelp.org/evaluatn/fnl_eval.htm \(14 of 15\)10/12/20\ 05 5:05:24 AM)Tj ET EMC endstream endobj 238 0 obj(Basic Guide to Program Evaluation) endobj 239 0 obj<> endobj 240 0 obj<> endobj 241 0 obj<> endobj 242 0 obj<> endobj 243 0 obj[240 0 R] endobj 244 0 obj(http://www.managementhelp.org/evaluatn/fnl_eval.htm) endobj 245 0 obj([évY\($) endobj 246 0 obj<> endobj 247 0 obj<> endobj 248 0 obj(>o^,cviɳl) endobj 249 0 obj 1 endobj 250 0 obj<> endobj 251 0 obj<> endobj 252 0 obj<>stream Basic Guide to Program Evaluation endstream endobj xref 0 253 0000000004 65535 f 0000000016 00000 n 0000000162 00000 n 0000000223 00000 n 0000000128 00001 f 0000000382 00000 n 0000000402 00000 n 0000000470 00000 n 0000000514 00000 n 0000000532 00000 n 0000000631 00000 n 0000000651 00000 n 0000000976 00000 n 0000001044 00000 n 0000001078 00000 n 0000001160 00000 n 0000001211 00000 n 0000001267 00000 n 0000001338 00000 n 0000001409 00000 n 0000001456 00000 n 0000001476 00000 n 0000001913 00000 n 0000001934 00000 n 0000003181 00000 n 0000003254 00000 n 0000003288 00000 n 0000003376 00000 n 0000003427 00000 n 0000003531 00000 n 0000003576 00000 n 0000003625 00000 n 0000004116 00000 n 0000004330 00000 n 0000005055 00000 n 0000005096 00000 n 0000005269 00000 n 0000005293 00000 n 0000005485 00000 n 0000005616 00000 n 0000005692 00000 n 0000005781 00000 n 0000006972 00000 n 0000007199 00000 n 0000007382 00000 n 0000007564 00000 n 0000007735 00000 n 0000007918 00000 n 0000008101 00000 n 0000008280 00000 n 0000008452 00000 n 0000008656 00000 n 0000008854 00000 n 0000009045 00000 n 0000009232 00000 n 0000009493 00000 n 0000009746 00000 n 0000009939 00000 n 0000010131 00000 n 0000010317 00000 n 0000010566 00000 n 0000010813 00000 n 0000011062 00000 n 0000011311 00000 n 0000011560 00000 n 0000011809 00000 n 0000012058 00000 n 0000012307 00000 n 0000012562 00000 n 0000012817 00000 n 0000013066 00000 n 0000013315 00000 n 0000013495 00000 n 0000013722 00000 n 0000013746 00000 n 0000013933 00000 n 0000013971 00000 n 0000014171 00000 n 0000014355 00000 n 0000014401 00000 n 0000014601 00000 n 0000014625 00000 n 0000014671 00000 n 0000014871 00000 n 0000014895 00000 n 0000014941 00000 n 0000014984 00000 n 0000015171 00000 n 0000015195 00000 n 0000015241 00000 n 0000015285 00000 n 0000015512 00000 n 0000015536 00000 n 0000015789 00000 n 0000015813 00000 n 0000016027 00000 n 0000016051 00000 n 0000016226 00000 n 0000016250 00000 n 0000016452 00000 n 0000016476 00000 n 0000016732 00000 n 0000016782 00000 n 0000017021 00000 n 0000017271 00000 n 0000017527 00000 n 0000017574 00000 n 0000017790 00000 n 0000017815 00000 n 0000018012 00000 n 0000018046 00000 n 0000018240 00000 n 0000018288 00000 n 0000018504 00000 n 0000018529 00000 n 0000018722 00000 n 0000018788 00000 n 0000018976 00000 n 0000019164 00000 n 0000019352 00000 n 0000019535 00000 n 0000019723 00000 n 0000019771 00000 n 0000019986 00000 n 0000020123 00000 n 0000020217 00000 n 0000020307 00000 n 0000020398 00000 n 0000000129 00001 f 0000000134 00001 f 0000020494 00000 n 0000020519 00000 n 0000020567 00000 n 0000020784 00000 n 0000000135 00001 f 0000000144 00001 f 0000020876 00000 n 0000020901 00000 n 0000020949 00000 n 0000020997 00000 n 0000021252 00000 n 0000021286 00000 n 0000021536 00000 n 0000021584 00000 n 0000000145 00001 f 0000000149 00001 f 0000021773 00000 n 0000021798 00000 n 0000021841 00000 n 0000000150 00001 f 0000000169 00001 f 0000022045 00000 n 0000022070 00000 n 0000022118 00000 n 0000022166 00000 n 0000022367 00000 n 0000022481 00000 n 0000022529 00000 n 0000022717 00000 n 0000022905 00000 n 0000023102 00000 n 0000023304 00000 n 0000023491 00000 n 0000023671 00000 n 0000023859 00000 n 0000024047 00000 n 0000024233 00000 n 0000024421 00000 n 0000024599 00000 n 0000000170 00001 f 0000000190 00001 f 0000024853 00000 n 0000024878 00000 n 0000025110 00000 n 0000025192 00000 n 0000025263 00000 n 0000025330 00000 n 0000025401 00000 n 0000025603 00000 n 0000025926 00000 n 0000026069 00000 n 0000026220 00000 n 0000026420 00000 n 0000026440 00000 n 0000026651 00000 n 0000026911 00000 n 0000027071 00000 n 0000027267 00000 n 0000027448 00000 n 0000027632 00000 n 0000000191 00001 f 0000000192 00001 f 0000000193 00001 f 0000000194 00001 f 0000000195 00001 f 0000000196 00001 f 0000000197 00001 f 0000000198 00001 f 0000000199 00001 f 0000000200 00001 f 0000000201 00001 f 0000000202 00001 f 0000000203 00001 f 0000000204 00001 f 0000000205 00001 f 0000000206 00001 f 0000000207 00001 f 0000000000 00001 f 0000027814 00000 n 0000027836 00000 n 0000034585 00000 n 0000034607 00000 n 0000039619 00000 n 0000039641 00000 n 0000044360 00000 n 0000044382 00000 n 0000048676 00000 n 0000048698 00000 n 0000052830 00000 n 0000052852 00000 n 0000055335 00000 n 0000055357 00000 n 0000059908 00000 n 0000059930 00000 n 0000064573 00000 n 0000064595 00000 n 0000073443 00000 n 0000073466 00000 n 0000087371 00000 n 0000087393 00000 n 0000091793 00000 n 0000091815 00000 n 0000095928 00000 n 0000095950 00000 n 0000100664 00000 n 0000100686 00000 n 0000104752 00000 n 0000104774 00000 n 0000109647 00000 n 0000109699 00000 n 0000109737 00000 n 0000109825 00000 n 0000109856 00000 n 0000109966 00000 n 0000109992 00000 n 0000110062 00000 n 0000110098 00000 n 0000110319 00000 n 0000110372 00000 n 0000110407 00000 n 0000110426 00000 n 0000110521 00000 n 0000110636 00000 n trailer <<3c8f2e33554ca144a289b9ddeb08b7ab>]>> startxref 113969 %%EOF